From b87676bad262d2f9e6d8d95c11f534064b6d6a9f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:16:39 +0100 Subject: [PATCH 01/37] refactor(cli): run sqb clone natively --- crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- .../src/clone_text/_helpers/mod.rs | 1 + .../src/clone_text/_helpers/quoted.rs | 57 +++ .../sqlbuild-core/src/clone_text/constants.rs | 24 ++ crates/sqlbuild-core/src/clone_text/errors.rs | 8 + .../src/clone_text/main/clone_exit_code.rs | 13 + .../main/clone_relation_flow_text.rs | 19 + .../sqlbuild-core/src/clone_text/main/mod.rs | 9 + .../main/render_clone_completion.rs | 58 +++ ...nder_clone_fingerprint_interrupted_line.rs | 33 ++ .../render_clone_fingerprint_progress_line.rs | 18 + .../clone_text/main/render_clone_header.rs | 22 + .../clone_text/main/render_clone_item_line.rs | 50 +++ .../clone_text/main/validate_clone_policy.rs | 29 ++ .../main/validate_clone_target_names.rs | 31 ++ crates/sqlbuild-core/src/clone_text/mod.rs | 10 + crates/sqlbuild-core/src/clone_text/models.rs | 42 ++ crates/sqlbuild-core/src/clone_text/tests.rs | 6 + .../src/clone_text/tests/helpers.rs | 29 ++ .../src/clone_text/tests/test_clone_text.rs | 292 +++++++++++++ .../src/clone_text/tests/test_types.rs | 39 ++ crates/sqlbuild-core/src/lib.rs | 1 + .../bindings/_helpers/clone/clone_command.rs | 366 +++++++++++++++++ .../_helpers/clone/clone_execution.rs | 382 ++++++++++++++++++ .../bindings/_helpers/clone/clone_progress.rs | 129 ++++++ .../src/bindings/_helpers/clone/mod.rs | 4 + .../bindings/_helpers/clone/registration.rs | 9 + .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/main/register.rs | 3 +- src/sqlbuild/_native.pyi | 2 + .../cli/commands/_helpers/clone/__init__.py | 1 - .../commands/_helpers/clone/connections.py | 60 --- .../cli/commands/_helpers/clone/execution.py | 254 ------------ .../_helpers/clone/fingerprint_progress.py | 58 --- .../cli/commands/_helpers/clone/invocation.py | 76 ---- .../cli/commands/_helpers/clone/output.py | 194 --------- .../cli/commands/_helpers/clone/outputs.py | 93 ----- .../cli/commands/_helpers/clone/planning.py | 80 ---- .../cli/commands/_helpers/clone/validation.py | 52 --- .../cli/commands/main/inspection/_clone.py | 60 --- src/sqlbuild/cli/commands/models.py | 38 -- .../pipeline/main/{clone.py => _clone.py} | 0 .../presentation/_helpers/structure.py | 13 - src/sqlbuild/presentation/main/phase_line.py | 12 - .../cli/commands/_helpers/clone/__init__.py | 1 - .../commands/_helpers/clone/_test_types.py | 57 --- .../_helpers/clone/test_connections.py | 84 ---- .../clone/test_fingerprint_progress.py | 56 --- .../commands/_helpers/clone/test_output.py | 228 ----------- 49 files changed, 1688 insertions(+), 1420 deletions(-) create mode 100644 crates/sqlbuild-core/src/clone_text/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/clone_text/_helpers/quoted.rs create mode 100644 crates/sqlbuild-core/src/clone_text/constants.rs create mode 100644 crates/sqlbuild-core/src/clone_text/errors.rs create mode 100644 crates/sqlbuild-core/src/clone_text/main/clone_exit_code.rs create mode 100644 crates/sqlbuild-core/src/clone_text/main/clone_relation_flow_text.rs create mode 100644 crates/sqlbuild-core/src/clone_text/main/mod.rs create mode 100644 crates/sqlbuild-core/src/clone_text/main/render_clone_completion.rs create mode 100644 crates/sqlbuild-core/src/clone_text/main/render_clone_fingerprint_interrupted_line.rs create mode 100644 crates/sqlbuild-core/src/clone_text/main/render_clone_fingerprint_progress_line.rs create mode 100644 crates/sqlbuild-core/src/clone_text/main/render_clone_header.rs create mode 100644 crates/sqlbuild-core/src/clone_text/main/render_clone_item_line.rs create mode 100644 crates/sqlbuild-core/src/clone_text/main/validate_clone_policy.rs create mode 100644 crates/sqlbuild-core/src/clone_text/main/validate_clone_target_names.rs create mode 100644 crates/sqlbuild-core/src/clone_text/mod.rs create mode 100644 crates/sqlbuild-core/src/clone_text/models.rs create mode 100644 crates/sqlbuild-core/src/clone_text/tests.rs create mode 100644 crates/sqlbuild-core/src/clone_text/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/clone_text/tests/test_clone_text.rs create mode 100644 crates/sqlbuild-core/src/clone_text/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/clone/__init__.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/clone/connections.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/clone/execution.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/clone/fingerprint_progress.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/clone/invocation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/clone/output.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/clone/outputs.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/clone/planning.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/clone/validation.py delete mode 100644 src/sqlbuild/cli/commands/main/inspection/_clone.py rename src/sqlbuild/compiler/pipeline/main/{clone.py => _clone.py} (100%) delete mode 100644 src/sqlbuild/presentation/main/phase_line.py delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/clone/__init__.py delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_connections.py delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_fingerprint_progress.py delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_output.py diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 450b6d7bf1..e3fe296a2a 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -387,9 +387,9 @@ const CLONE_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const CLONE_ROUTE: Route = routed( "run_clone", - ("sqlbuild.cli.commands.main.inspection._clone", "run_clone"), + ("sqlbuild._native", "run_clone_command"), &CLONE_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const DIFF_FIELDS: [Field; 35] = [ diff --git a/crates/sqlbuild-core/src/clone_text/_helpers/mod.rs b/crates/sqlbuild-core/src/clone_text/_helpers/mod.rs new file mode 100644 index 0000000000..9446b134c0 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod quoted; diff --git a/crates/sqlbuild-core/src/clone_text/_helpers/quoted.rs b/crates/sqlbuild-core/src/clone_text/_helpers/quoted.rs new file mode 100644 index 0000000000..25f981ae02 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/_helpers/quoted.rs @@ -0,0 +1,57 @@ +//! Quoted identifier spans in a relation name. + +/// The end of a doubled-escape quoted identifier opening at `start`, as a backtracking regex finds it. +fn quoted_end(chars: &[char], start: usize) -> Option { + let close: char = match chars[start] { + '"' => '"', + '`' => '`', + '[' => ']', + _ => return None, + }; + let mut position: usize = start + 1; + let mut last_pair: Option = None; + while position < chars.len() { + if chars[position] != close { + position += 1; + continue; + } + if chars.get(position + 1) == Some(&close) { + last_pair = Some(position); + position += 2; + continue; + } + return Some(position + 1); + } + last_pair.map(|pair| pair + 1) +} + +/// The relation split into `(text, quoted)` segments in order. +fn quoted_segments(relation: &str) -> Vec<(String, bool)> { + let chars: Vec = relation.chars().collect(); + let mut segments: Vec<(String, bool)> = Vec::new(); + let mut plain: String = String::new(); + let mut position: usize = 0; + while position < chars.len() { + match quoted_end(&chars, position) { + Some(end) => { + segments.push((std::mem::take(&mut plain), false)); + segments.push((chars[position..end].iter().collect(), true)); + position = end; + } + None => { + plain.push(chars[position]); + position += 1; + } + } + } + segments.push((plain, false)); + segments +} + +/// Lowercase ordinary identifiers while preserving quoted identifier case. +pub(crate) fn clone_relation_display_name(relation: &str) -> String { + quoted_segments(relation) + .into_iter() + .map(|(text, quoted)| if quoted { text } else { text.to_lowercase() }) + .collect() +} diff --git a/crates/sqlbuild-core/src/clone_text/constants.rs b/crates/sqlbuild-core/src/clone_text/constants.rs new file mode 100644 index 0000000000..9a30eab26f --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/constants.rs @@ -0,0 +1,24 @@ +//! Clone output labels. + +use crate::clone_text::models::CloneStatusKind; + +/// The width of the longest clone action value. +pub(crate) const ACTION_WIDTH: usize = "skipped_missing_dependency".len(); +pub(crate) const FINGERPRINT_LABEL: &str = "fingerprint"; +pub(crate) const FINGERPRINT_LABEL_WIDTH: usize = 10; +pub(crate) const PENDING_IDENTITY_LIMIT: usize = 10; + +/// Action footer labels and the action value each counts. +pub(crate) const ACTION_SUMMARY: [(&str, &str); 4] = [ + ("CLONED", "cloned"), + ("COPIED", "copied"), + ("RECREATED_VIEWS", "recreated_view"), + ("RECREATED_FUNCTIONS", "recreated_function"), +]; + +/// Status footer labels and the status each counts. +pub(crate) const STATUS_SUMMARY: [(&str, CloneStatusKind); 3] = [ + ("PASS", CloneStatusKind::Success), + ("WARN", CloneStatusKind::Warning), + ("FAIL", CloneStatusKind::Failed), +]; diff --git a/crates/sqlbuild-core/src/clone_text/errors.rs b/crates/sqlbuild-core/src/clone_text/errors.rs new file mode 100644 index 0000000000..03c08d6809 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/errors.rs @@ -0,0 +1,8 @@ +//! Clone request validation errors. + +/// A clone request the targets do not allow, with its diagnostic code. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CloneTargetError { + pub code: &'static str, + pub message: String, +} diff --git a/crates/sqlbuild-core/src/clone_text/main/clone_exit_code.rs b/crates/sqlbuild-core/src/clone_text/main/clone_exit_code.rs new file mode 100644 index 0000000000..c4b017d461 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/clone_exit_code.rs @@ -0,0 +1,13 @@ +//! The exit code of `sqb clone`. + +use crate::clone_text::models::{CloneItemText, CloneStatusKind}; + +/// 1 when any item failed, else 0. +#[must_use] +pub fn clone_exit_code(items: &[CloneItemText]) -> i32 { + i32::from( + items + .iter() + .any(|item| item.status == CloneStatusKind::Failed), + ) +} diff --git a/crates/sqlbuild-core/src/clone_text/main/clone_relation_flow_text.rs b/crates/sqlbuild-core/src/clone_text/main/clone_relation_flow_text.rs new file mode 100644 index 0000000000..b2ae1c8b00 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/clone_relation_flow_text.rs @@ -0,0 +1,19 @@ +//! The plain origin-to-destination text of one clone item. + +use crate::clone_text::_helpers::quoted::clone_relation_display_name; + +/// `origin -> destination`, or the lowercased name when either relation is unknown. +#[must_use] +pub fn clone_relation_flow_text( + name: &str, + (origin, destination): (Option<&str>, Option<&str>), +) -> String { + match (origin, destination) { + (Some(origin), Some(destination)) => format!( + "{} -> {}", + clone_relation_display_name(origin), + clone_relation_display_name(destination) + ), + _ => name.to_lowercase(), + } +} diff --git a/crates/sqlbuild-core/src/clone_text/main/mod.rs b/crates/sqlbuild-core/src/clone_text/main/mod.rs new file mode 100644 index 0000000000..6f6ec74604 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/mod.rs @@ -0,0 +1,9 @@ +pub mod clone_exit_code; +pub mod clone_relation_flow_text; +pub mod render_clone_completion; +pub mod render_clone_fingerprint_interrupted_line; +pub mod render_clone_fingerprint_progress_line; +pub mod render_clone_header; +pub mod render_clone_item_line; +pub mod validate_clone_policy; +pub mod validate_clone_target_names; diff --git a/crates/sqlbuild-core/src/clone_text/main/render_clone_completion.rs b/crates/sqlbuild-core/src/clone_text/main/render_clone_completion.rs new file mode 100644 index 0000000000..9d31dc66ca --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/render_clone_completion.rs @@ -0,0 +1,58 @@ +//! The final clone output: non-success messages, then the completion line with counts. + +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, CompletionState, SummaryCount}; +use crate::clone_text::constants::{ACTION_SUMMARY, STATUS_SUMMARY}; +use crate::clone_text::models::{CloneItemText, CloneStatusKind}; + +fn count(items: &[CloneItemText], predicate: impl Fn(&CloneItemText) -> bool) -> i64 { + i64::try_from(items.iter().filter(|item| predicate(item)).count()).unwrap_or(i64::MAX) +} + +/// Messages of items that did not succeed, a blank line, and the completion summary. +#[must_use] +pub fn render_clone_completion( + items: &[CloneItemText], + elapsed_seconds: Option, + style: CliStyle, +) -> String { + let mut text: String = String::new(); + for item in items { + if let (Some(message), false) = (&item.message, item.status == CloneStatusKind::Success) { + text.push_str(&format!( + " {} {}\n", + style.object_name(&item.name), + style.muted(message) + )); + } + } + let mut counts: Vec = ACTION_SUMMARY + .iter() + .map(|(label, action)| SummaryCount { + label: (*label).to_owned(), + value: count(items, |item| item.action == *action), + }) + .collect(); + counts.extend(STATUS_SUMMARY.iter().map(|(label, status)| SummaryCount { + label: (*label).to_owned(), + value: count(items, |item| item.status == *status), + })); + counts.push(SummaryCount { + label: "TOTAL".to_owned(), + value: count(items, |_| true), + }); + let elapsed: Option = elapsed_seconds.map(|seconds| format!("({seconds:.2}s)")); + let footer: String = format_summary_footer(style, &counts, elapsed.as_deref()); + let warnings: i64 = count(items, |item| item.status == CloneStatusKind::Warning); + let failures: i64 = count(items, |item| item.status == CloneStatusKind::Failed); + let (state, label) = match (warnings, failures) { + (0, 0) => (CompletionState::Ok, "Completed successfully"), + (_, 0) => (CompletionState::Warn, "Completed with warnings"), + _ => (CompletionState::Fail, "Completed with errors"), + }; + text.push('\n'); + text.push_str(&format_completion_line(style, state, label, Some(&footer))); + text.push('\n'); + text +} diff --git a/crates/sqlbuild-core/src/clone_text/main/render_clone_fingerprint_interrupted_line.rs b/crates/sqlbuild-core/src/clone_text/main/render_clone_fingerprint_interrupted_line.rs new file mode 100644 index 0000000000..5b5a2a0672 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/render_clone_fingerprint_interrupted_line.rs @@ -0,0 +1,33 @@ +//! Bounded context for interrupted clone fingerprint propagation. + +use crate::cli_text::models::CliStyle; +use crate::clone_text::constants::{ + FINGERPRINT_LABEL, FINGERPRINT_LABEL_WIDTH, PENDING_IDENTITY_LIMIT, +}; +use crate::clone_text::models::FingerprintProgress; + +/// The confirmed count and up to ten pending identities; none once everything was confirmed. +#[must_use] +pub fn render_clone_fingerprint_interrupted_line( + progress: &FingerprintProgress, + style: CliStyle, +) -> Option { + if progress.total <= progress.completed { + return None; + } + let shown: usize = progress + .pending_identities + .len() + .min(PENDING_IDENTITY_LIMIT); + let remaining: usize = progress.pending_identities.len() - shown; + let mut pending: String = progress.pending_identities[..shown].join(", "); + if remaining > 0 { + pending = format!("{pending} (+{remaining} more)"); + } + Some(format!( + " {FINGERPRINT_LABEL: String { + let width: usize = total.to_string().len(); + format!( + " {FINGERPRINT_LABEL:width$}/{total}] propagated {}", + style.status("OK") + ) +} diff --git a/crates/sqlbuild-core/src/clone_text/main/render_clone_header.rs b/crates/sqlbuild-core/src/clone_text/main/render_clone_header.rs new file mode 100644 index 0000000000..9ec2a9a210 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/render_clone_header.rs @@ -0,0 +1,22 @@ +//! The header naming the clone origin, destination and resource count. + +use crate::cli_text::models::CliStyle; + +/// `sqb clone origin= destination= ( resources)`. +#[must_use] +pub fn render_clone_header( + (origin, destination): (&str, &str), + total: usize, + style: CliStyle, +) -> String { + let plural: &str = if total == 1 { "" } else { "s" }; + format!( + "{} {}{} {}{} {}", + style.title("sqb clone"), + style.label("origin="), + style.value(origin), + style.label("destination="), + style.value(destination), + style.muted(&format!("({total} resource{plural})")) + ) +} diff --git a/crates/sqlbuild-core/src/clone_text/main/render_clone_item_line.rs b/crates/sqlbuild-core/src/clone_text/main/render_clone_item_line.rs new file mode 100644 index 0000000000..237e31d9cc --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/render_clone_item_line.rs @@ -0,0 +1,50 @@ +//! One streamed clone progress line. + +use crate::cli_text::models::CliStyle; +use crate::clone_text::_helpers::quoted::clone_relation_display_name; +use crate::clone_text::constants::ACTION_WIDTH; +use crate::clone_text::main::clone_relation_flow_text::clone_relation_flow_text; +use crate::clone_text::models::{CloneItemText, CloneStatusKind}; + +/// Position, action, origin to destination padded to `relation_width`, status and duration. +#[must_use] +pub fn render_clone_item_line( + (index, total): (usize, usize), + item: &CloneItemText, + relation_width: usize, + style: CliStyle, +) -> String { + let position: String = format!("{index:>width$}/{total}", width = total.to_string().len()); + let relations = ( + item.origin_relation.as_deref(), + item.destination_relation.as_deref(), + ); + let plain_flow: String = clone_relation_flow_text(&item.name, relations); + let flow: String = match relations { + (Some(origin), Some(destination)) => format!( + "{} {} {}", + style.object_name(&clone_relation_display_name(origin)), + style.muted("->"), + style.object_name(&clone_relation_display_name(destination)) + ), + _ => style.object_name(&plain_flow), + }; + let plain_width: usize = plain_flow.chars().count(); + let relation_padding: String = " ".repeat(relation_width.max(plain_width) - plain_width); + let action_padding: String = + " ".repeat(ACTION_WIDTH.saturating_sub(item.action.chars().count())); + let status: &str = match item.status { + CloneStatusKind::Warning => "WARN", + CloneStatusKind::Failed => "FAIL", + CloneStatusKind::Success => "OK", + }; + let duration: String = item.duration_seconds.map_or_else(String::new, |seconds| { + format!(" {}", style.muted(&format!("{seconds:.2}s"))) + }); + format!( + " {} {}{action_padding} {flow}{relation_padding} {}{duration}", + style.muted(&position), + style.accent(&item.action), + style.status(status) + ) +} diff --git a/crates/sqlbuild-core/src/clone_text/main/validate_clone_policy.rs b/crates/sqlbuild-core/src/clone_text/main/validate_clone_policy.rs new file mode 100644 index 0000000000..bdd4e2abf0 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/validate_clone_policy.rs @@ -0,0 +1,29 @@ +//! Check that the targets' clone policies allow the requested direction. + +use crate::clone_text::errors::CloneTargetError; + +/// C404 when the origin may not be cloned from, C405 when the destination may not be cloned to. +pub fn validate_clone_policy( + (origin, destination): (&str, &str), + (origin_allowed, destination_allowed): (bool, bool), +) -> Result<(), CloneTargetError> { + if !origin_allowed { + return Err(CloneTargetError { + code: "C404", + message: format!( + "target '{origin}' is not allowed as a clone origin target; \ + set targets.{origin}.clone.allow_as_clone_origin = true" + ), + }); + } + if !destination_allowed { + return Err(CloneTargetError { + code: "C405", + message: format!( + "target '{destination}' is not allowed as a clone destination target; \ + set targets.{destination}.clone.allow_as_clone_destination = true" + ), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/clone_text/main/validate_clone_target_names.rs b/crates/sqlbuild-core/src/clone_text/main/validate_clone_target_names.rs new file mode 100644 index 0000000000..1f6a2b1bf8 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/validate_clone_target_names.rs @@ -0,0 +1,31 @@ +//! Check that clone origin and destination name distinct, known targets. + +use std::collections::BTreeSet; + +use crate::clone_text::errors::CloneTargetError; + +/// C401 for the same target, C402 for an unknown origin, C403 for an unknown destination. +pub fn validate_clone_target_names( + (origin, destination): (&str, &str), + known: &BTreeSet, +) -> Result<(), CloneTargetError> { + if origin == destination { + return Err(CloneTargetError { + code: "C401", + message: "clone requires different --from and --to targets".to_owned(), + }); + } + if !known.contains(origin) { + return Err(CloneTargetError { + code: "C402", + message: format!("unknown target '{origin}'"), + }); + } + if !known.contains(destination) { + return Err(CloneTargetError { + code: "C403", + message: format!("unknown target '{destination}'"), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/clone_text/mod.rs b/crates/sqlbuild-core/src/clone_text/mod.rs new file mode 100644 index 0000000000..8ea04437d8 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/mod.rs @@ -0,0 +1,10 @@ +//! `sqb clone`: target validation, streamed progress lines, the completion summary and exit code. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/clone_text/models.rs b/crates/sqlbuild-core/src/clone_text/models.rs new file mode 100644 index 0000000000..fa27816787 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/models.rs @@ -0,0 +1,42 @@ +//! Clone item results as the command renders them. + +/// The outcome of one cloned resource. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CloneStatusKind { + Success, + Warning, + Failed, +} + +impl CloneStatusKind { + /// The status from its value, treating anything unknown as success. + #[must_use] + pub fn from_value(value: &str) -> Self { + match value { + "warning" => Self::Warning, + "failed" => Self::Failed, + _ => Self::Success, + } + } +} + +/// One clone item result. +#[derive(Clone, Debug, PartialEq)] +pub struct CloneItemText { + pub name: String, + /// The `CloneAction` value, such as `copied`. + pub action: String, + pub status: CloneStatusKind, + pub message: Option, + pub origin_relation: Option, + pub destination_relation: Option, + pub duration_seconds: Option, +} + +/// Fingerprint propagation progress: confirmed count, total, and identities still pending. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FingerprintProgress { + pub completed: i64, + pub total: i64, + pub pending_identities: Vec, +} diff --git a/crates/sqlbuild-core/src/clone_text/tests.rs b/crates/sqlbuild-core/src/clone_text/tests.rs new file mode 100644 index 0000000000..1f95e219da --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_clone_text.rs"] +mod clone_text; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/clone_text/tests/helpers.rs b/crates/sqlbuild-core/src/clone_text/tests/helpers.rs new file mode 100644 index 0000000000..924194644c --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/tests/helpers.rs @@ -0,0 +1,29 @@ +use crate::clone_text::models::{CloneItemText, CloneStatusKind}; + +/// An item with an action and status and no relations or message. +pub(super) fn item(name: &str, action: &str, status: CloneStatusKind) -> CloneItemText { + CloneItemText { + name: name.to_owned(), + action: action.to_owned(), + status, + message: None, + origin_relation: None, + destination_relation: None, + duration_seconds: None, + } +} + +/// An item with a message. +pub(super) fn with_message(item: CloneItemText, message: &str) -> CloneItemText { + CloneItemText { + message: Some(message.to_owned()), + ..item + } +} + +/// Twelve pending model identities. +pub(super) fn pending_identities() -> Vec { + (0..12) + .map(|index| format!("model:orders_{index}")) + .collect() +} diff --git a/crates/sqlbuild-core/src/clone_text/tests/test_clone_text.rs b/crates/sqlbuild-core/src/clone_text/tests/test_clone_text.rs new file mode 100644 index 0000000000..65d91bb78d --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/tests/test_clone_text.rs @@ -0,0 +1,292 @@ +use std::collections::BTreeSet; + +use crate::cli_text::models::CliStyle; +use crate::clone_text::_helpers::quoted::clone_relation_display_name; +use crate::clone_text::main::clone_exit_code::clone_exit_code; +use crate::clone_text::main::render_clone_completion::render_clone_completion; +use crate::clone_text::main::render_clone_fingerprint_interrupted_line::render_clone_fingerprint_interrupted_line; +use crate::clone_text::main::render_clone_fingerprint_progress_line::render_clone_fingerprint_progress_line; +use crate::clone_text::main::render_clone_item_line::render_clone_item_line; +use crate::clone_text::main::validate_clone_policy::validate_clone_policy; +use crate::clone_text::main::validate_clone_target_names::validate_clone_target_names; +use crate::clone_text::models::{CloneItemText, CloneStatusKind, FingerprintProgress}; +use crate::clone_text::tests::helpers::{item, pending_identities, with_message}; +use crate::clone_text::tests::test_types::{ + CompletionTestCase, DisplayNameTestCase, FingerprintTestCase, ItemLineTestCase, TargetTestCase, +}; + +#[test] +fn given_relation_when_displaying_then_lowercases_only_unquoted_parts() { + let test_cases = [ + DisplayNameTestCase { + description: "plain identifiers are lowercased", + relation: "PROD.Fact_Orders", + expected_name: "prod.fact_orders", + }, + DisplayNameTestCase { + description: "double, backtick and bracket quotes keep their case", + relation: "DB.\"Mixed\"\"Case\".`Tick`.[Br]]ack]", + expected_name: "db.\"Mixed\"\"Case\".`Tick`.[Br]]ack]", + }, + DisplayNameTestCase { + description: "an unclosed quote is ordinary text", + relation: "A.\"Open", + expected_name: "a.\"open", + }, + DisplayNameTestCase { + description: "a trailing doubled quote backtracks to the first closing quote", + relation: "\"Ab\"\"C", + expected_name: "\"Ab\"\"c", + }, + ]; + + for test_case in test_cases { + assert_eq!( + clone_relation_display_name(test_case.relation), + test_case.expected_name, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_item_when_rendering_line_then_aligns_action_and_relations() { + let test_cases = [ + ItemLineTestCase { + description: "relations, duration and padding to the widest flow", + position: (2, 12), + item: CloneItemText { + origin_relation: Some("PROD.Orders".to_owned()), + destination_relation: Some("dev.orders".to_owned()), + duration_seconds: Some(0.125), + ..item("orders", "copied", CloneStatusKind::Success) + }, + relation_width: 30, + use_color: false, + expected_line: " 2/12 copied prod.orders -> dev.orders OK 0.12s", + }, + ItemLineTestCase { + description: "quoted identifiers keep their case in the flow", + position: (5, 5), + item: CloneItemText { + origin_relation: Some("PROD.DBO.[My]]Table]".to_owned()), + destination_relation: Some("DEV.MAIN.\"Orders\"".to_owned()), + ..item("orders", "cloned", CloneStatusKind::Success) + }, + relation_width: 0, + use_color: false, + expected_line: " 5/5 cloned prod.dbo.[My]]Table] -> dev.main.\"Orders\" OK", + }, + ItemLineTestCase { + description: "missing relations fall back to the lowercased name", + position: (1, 1), + item: item("Stg", "warning_missing_source", CloneStatusKind::Warning), + relation_width: 0, + use_color: false, + expected_line: " 1/1 warning_missing_source stg WARN", + }, + ItemLineTestCase { + description: "colour styles position, action, arrow and status", + position: (1, 1), + item: CloneItemText { + origin_relation: Some("a.x".to_owned()), + destination_relation: Some("b.x".to_owned()), + ..item("x", "failed", CloneStatusKind::Failed) + }, + relation_width: 0, + use_color: true, + expected_line: " \u{1b}[2m1/1\u{1b}[0m \u{1b}[34mfailed\u{1b}[0m a.x \u{1b}[2m->\u{1b}[0m b.x \u{1b}[38;5;167mFAIL\u{1b}[0m", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_clone_item_line( + test_case.position, + &test_case.item, + test_case.relation_width, + CliStyle { + use_color: test_case.use_color + }, + ), + test_case.expected_line, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_items_when_completing_then_reports_messages_counts_and_exit_code() { + let test_cases = [ + CompletionTestCase { + description: "a failure reports errors and messages of non-success items", + items: vec![ + item("fact_orders", "cloned", CloneStatusKind::Success), + with_message( + item( + "stg_orders", + "warning_missing_source", + CloneStatusKind::Warning, + ), + "missing in origin environment", + ), + with_message( + item("dim_customers", "failed", CloneStatusKind::Failed), + "boom", + ), + with_message( + item("add_tax", "recreated_function", CloneStatusKind::Success), + "ignored", + ), + ], + elapsed_seconds: None, + use_color: false, + expected_text: " stg_orders missing in origin environment\n dim_customers boom\n\n\ + \u{2717} Completed with errors CLONED=1 COPIED=0 RECREATED_VIEWS=0 \ + RECREATED_FUNCTIONS=1 PASS=2 WARN=1 FAIL=1 TOTAL=4\n", + expected_exit_code: 1, + }, + CompletionTestCase { + description: "success shows the doubly parenthesised elapsed time", + items: vec![item("orders", "copied", CloneStatusKind::Success)], + elapsed_seconds: Some(1.5), + use_color: false, + expected_text: "\n\u{2713} Completed successfully CLONED=0 COPIED=1 RECREATED_VIEWS=0 \ + RECREATED_FUNCTIONS=0 PASS=1 WARN=0 FAIL=0 TOTAL=1 ((1.50s))\n", + expected_exit_code: 0, + }, + CompletionTestCase { + description: "colour styles the failure glyph and label", + items: vec![item("orders", "failed", CloneStatusKind::Failed)], + elapsed_seconds: None, + use_color: true, + expected_text: "\n\u{1b}[38;5;167m\u{2717}\u{1b}[0m \u{1b}[38;5;167m\u{1b}[1mCompleted with errors\u{1b}[0m \ + \u{1b}[2mCLONED=\u{1b}[0m\u{1b}[34m0\u{1b}[0m \u{1b}[2mCOPIED=\u{1b}[0m\u{1b}[34m0\u{1b}[0m \ + \u{1b}[2mRECREATED_VIEWS=\u{1b}[0m\u{1b}[34m0\u{1b}[0m \ + \u{1b}[2mRECREATED_FUNCTIONS=\u{1b}[0m\u{1b}[34m0\u{1b}[0m \ + \u{1b}[2mPASS=\u{1b}[0m\u{1b}[32m0\u{1b}[0m \u{1b}[2mWARN=\u{1b}[0m\u{1b}[33m0\u{1b}[0m \ + \u{1b}[2mFAIL=\u{1b}[0m\u{1b}[38;5;167m1\u{1b}[0m \u{1b}[2mTOTAL=\u{1b}[0m\u{1b}[34m1\u{1b}[0m\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + render_clone_completion( + &test_case.items, + test_case.elapsed_seconds, + CliStyle { + use_color: test_case.use_color + }, + ), + clone_exit_code(&test_case.items), + ), + ( + test_case.expected_text.to_owned(), + test_case.expected_exit_code + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_fingerprint_progress_when_rendering_then_reports_confirmed_and_pending() { + let test_cases = [ + FingerprintTestCase { + description: "progress pads the count and interruption bounds pending identities", + progress: FingerprintProgress { + completed: 50, + total: 386, + pending_identities: pending_identities(), + }, + expected_progress_line: " fingerprint[ 50/386] propagated OK", + expected_interrupted_line: Some( + " fingerprint[50/386] FAIL not confirmed: model:orders_0, \ + model:orders_1, model:orders_2, model:orders_3, model:orders_4, model:orders_5, \ + model:orders_6, model:orders_7, model:orders_8, model:orders_9 (+2 more)", + ), + }, + FingerprintTestCase { + description: "a fully confirmed propagation reports no interruption", + progress: FingerprintProgress { + completed: 3, + total: 3, + pending_identities: Vec::new(), + }, + expected_progress_line: " fingerprint[3/3] propagated OK", + expected_interrupted_line: None, + }, + ]; + + for test_case in test_cases { + let style: CliStyle = CliStyle { use_color: false }; + assert_eq!( + ( + render_clone_fingerprint_progress_line( + test_case.progress.completed, + test_case.progress.total, + style, + ), + render_clone_fingerprint_interrupted_line(&test_case.progress, style), + ), + ( + test_case.expected_progress_line.to_owned(), + test_case.expected_interrupted_line.map(str::to_owned), + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_targets_when_validating_then_reports_the_first_problem() { + let known: BTreeSet = ["dev", "prod"].into_iter().map(str::to_owned).collect(); + let test_cases = [ + TargetTestCase { + description: "known distinct targets with allowed policies pass", + targets: ("prod", "dev"), + policy: (true, true), + expected_codes: (None, None), + }, + TargetTestCase { + description: "the same target is rejected first", + targets: ("dev", "dev"), + policy: (true, true), + expected_codes: (Some("C401"), None), + }, + TargetTestCase { + description: "unknown origin", + targets: ("nope", "dev"), + policy: (false, true), + expected_codes: (Some("C402"), Some("C404")), + }, + TargetTestCase { + description: "unknown destination and denied destination policy", + targets: ("prod", "nope"), + policy: (true, false), + expected_codes: (Some("C403"), Some("C405")), + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + validate_clone_target_names(test_case.targets, &known) + .err() + .map(|error| error.code), + validate_clone_policy(test_case.targets, test_case.policy) + .err() + .map(|error| error.code), + ), + test_case.expected_codes, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/clone_text/tests/test_types.rs b/crates/sqlbuild-core/src/clone_text/tests/test_types.rs new file mode 100644 index 0000000000..904b525025 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/tests/test_types.rs @@ -0,0 +1,39 @@ +use crate::clone_text::models::{CloneItemText, FingerprintProgress}; + +pub(super) struct DisplayNameTestCase { + pub(super) description: &'static str, + pub(super) relation: &'static str, + pub(super) expected_name: &'static str, +} + +pub(super) struct ItemLineTestCase { + pub(super) description: &'static str, + pub(super) position: (usize, usize), + pub(super) item: CloneItemText, + pub(super) relation_width: usize, + pub(super) use_color: bool, + pub(super) expected_line: &'static str, +} + +pub(super) struct CompletionTestCase { + pub(super) description: &'static str, + pub(super) items: Vec, + pub(super) elapsed_seconds: Option, + pub(super) use_color: bool, + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} + +pub(super) struct FingerprintTestCase { + pub(super) description: &'static str, + pub(super) progress: FingerprintProgress, + pub(super) expected_progress_line: &'static str, + pub(super) expected_interrupted_line: Option<&'static str>, +} + +pub(super) struct TargetTestCase { + pub(super) description: &'static str, + pub(super) targets: (&'static str, &'static str), + pub(super) policy: (bool, bool), + pub(super) expected_codes: (Option<&'static str>, Option<&'static str>), +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 5afcac9c29..1be30d1a0f 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -3,6 +3,7 @@ #![forbid(unsafe_code)] pub mod cli_text; +pub mod clone_text; pub mod constants; pub mod cost_report; pub mod debug_report; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs new file mode 100644 index 0000000000..50470d396a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -0,0 +1,366 @@ +//! `sqb clone`: copy or clone selected relations from an origin target into a destination target. + +use std::collections::BTreeSet; +use std::time::Instant; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::main::format_phase_line::format_phase_line; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::clone_text::errors::CloneTargetError; +use sqlbuild_core::clone_text::main::clone_exit_code::clone_exit_code; +use sqlbuild_core::clone_text::main::render_clone_completion::render_clone_completion; +use sqlbuild_core::clone_text::main::render_clone_header::render_clone_header; +use sqlbuild_core::clone_text::main::validate_clone_policy::validate_clone_policy; +use sqlbuild_core::clone_text::main::validate_clone_target_names::validate_clone_target_names; +use sqlbuild_core::clone_text::models::CloneItemText; + +use crate::bindings::_helpers::clone::clone_execution::{ + destination_entry_count, execute_clone_plan, resource_types_by_name, +}; +use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const CLONE_PIPELINE_MODULE: &str = "sqlbuild.compiler.pipeline.main._clone"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const NO_DESTINATION: (&str, &str) = ( + "C406", + "clone requires --to when the project has no active target", +); +const NOTHING_TO_CLONE: (&str, &str) = + ("C407", "no cloneable resources found in the selected scope"); + +/// The discovered project, adapter and output context of one clone. +pub(crate) struct Invocation<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) discovered: Bound<'py, PyAny>, + pub(crate) adapter_name: String, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) destination: String, + pub(crate) style: CliStyle, + /// The stdout of the invocation, which progress keeps writing to while output is captured. + pub(crate) stream: Bound<'py, PyAny>, +} + +fn user_error(py: Python<'_>, (code, message): (&str, &str)) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", code)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn target_error(py: Python<'_>, error: CloneTargetError) -> PyErr { + user_error(py, (error.code, &error.message)) +} + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn progress_reporter<'py>( + stream: &Bound<'py, PyAny>, + style: CliStyle, +) -> PyResult> { + let py: Python<'py> = stream.py(); + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("stream", stream)?; + options.set_item("use_color", style.use_color)?; + imported(py, NATIVE_MODULE, "PlanningProgressReporter")?.call((), Some(&options)) +} + +fn lifecycle<'py>(py: Python<'py>, (kind, name): (&str, &str)) -> PyResult> { + call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", text(py, kind)), + ("operation_name", text(py, name)), + ], + ) +} + +fn target_names(discovered: &Bound<'_, PyAny>) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + for config in ["project_config", "local_config"] { + for name in discovered.getattr(config)?.getattr("targets")?.try_iter()? { + names.insert(name?.extract()?); + } + } + Ok(names) +} + +fn clone_policy(discovered: &Bound<'_, PyAny>, (target, flag): (&str, &str)) -> PyResult { + let py: Python<'_> = discovered.py(); + call_with( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("target_name", text(py, target)), + ], + )? + .getattr("clone")? + .getattr(flag)? + .is_truthy() +} + +/// Discover the project, resolve and validate the targets, and resolve the adapter. +fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult> { + let project_dir: Bound<'py, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, + }; + let use_color: bool = !request.getattr("no_color")?.extract::()? + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .extract::()?; + let style: CliStyle = CliStyle { use_color }; + let stream: Bound<'py, PyAny> = py.import("sys")?.getattr("stdout")?; + let progress: Bound<'py, PyAny> = progress_reporter(&stream, style)?; + progress.call_method1("start", ("Discovering project...",))?; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + progress.call_method1("complete", ("Discovered project.",))?; + call_with( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let mut destination: Bound<'py, PyAny> = request.getattr("destination_target_name")?; + if destination.is_none() { + destination = call_with( + &imported(py, TARGET_NAME_MODULE, "resolve_target_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("selected_target", py.None().into_bound(py)), + ], + )?; + } + if destination.is_none() { + return Err(user_error(py, NO_DESTINATION)); + } + let destination: String = destination.extract()?; + let origin: String = request.getattr("origin_target_name")?.extract()?; + let targets: (&str, &str) = (&origin, &destination); + validate_clone_target_names(targets, &target_names(&discovered)?) + .map_err(|error| target_error(py, error))?; + let policy: (bool, bool) = ( + clone_policy(&discovered, (&origin, "allow_as_clone_origin"))?, + clone_policy(&discovered, (&destination, "allow_as_clone_destination"))?, + ); + validate_clone_policy(targets, policy).map_err(|error| target_error(py, error))?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", project_dir.clone()), + ], + )?; + Ok(Invocation { + project_dir, + discovered, + adapter_name: adapter_name.extract()?, + adapter, + destination, + style, + stream, + }) +} + +/// Resolve the destination connection config and connect, returning both. +fn connect<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = request.py(); + let config: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_target_connection_config")?, + &[ + ("discovered_inputs", invocation.discovered.clone()), + ("project_dir", invocation.project_dir.clone()), + ("target_name", text(py, &invocation.destination)), + ("cli_vars", request.getattr("cli_vars")?), + ], + )?; + let progress: Bound<'py, PyAny> = progress_reporter(&invocation.stream, invocation.style)?; + progress.call_method1( + "start", + (format!("Connecting to {}...", invocation.adapter_name),), + )?; + let started: Instant = Instant::now(); + let connection: Bound<'py, PyAny> = within( + &lifecycle(py, ("clone", "clone_target_connection"))?, + |_| invocation.adapter.call_method1("connect", (&config,)), + )?; + let summary: String = format!( + "{} ({:.2}s)", + invocation.adapter_name, + started.elapsed().as_secs_f64() + ); + progress.call_method1( + "complete_styled", + (format_phase_line( + invocation.style, + true, + ("Warehouse connected", &summary), + ),), + )?; + Ok((config, connection)) +} + +/// Run the clone pipeline and refuse an empty selection. +fn prepare<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, + (config, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let progress: Bound<'py, PyAny> = progress_reporter(&invocation.stream, invocation.style)?; + let started: Instant = Instant::now(); + let pipeline: Bound<'py, PyAny> = + within(&lifecycle(py, ("project", "project_compile"))?, |_| { + progress.call_method1("on_progress", ("Preparing clone plan...",))?; + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.project_dir.clone()), + ("discovered_inputs", invocation.discovered.clone()), + ], + )?; + let destination_connection: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "ClonePipelineConnection")?, + &[("config", config.clone()), ("handle", connection.clone())], + )?; + let pipeline: Bound<'py, PyAny> = call_with( + &imported(py, CLONE_PIPELINE_MODULE, "run_clone_pipeline")?, + &[ + ("discovered_inputs", invocation.discovered.clone()), + ("adapter", invocation.adapter.clone()), + ("origin_target_name", request.getattr("origin_target_name")?), + ("destination_target_name", text(py, &invocation.destination)), + ("no_sql_validation", request.getattr("no_sql_validation")?), + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("cli_vars", request.getattr("cli_vars")?), + ("destination_connection", destination_connection), + ("external_sql_reference_resolver", resolver), + ], + )?; + progress.call_method1( + "on_progress", + (format!( + "Prepared clone plan. ({:.2}s)", + started.elapsed().as_secs_f64() + ),), + )?; + Ok(pipeline) + })?; + if destination_entry_count(&pipeline)? == 0 { + return Err(user_error(py, NOTHING_TO_CLONE)); + } + Ok(pipeline) +} + +/// Prepare, announce and execute the clone with an open destination connection. +fn clone<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, + (config, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>, f64)> { + let pipeline: Bound<'py, PyAny> = prepare(request, invocation, (config, connection))?; + let origin: String = request.getattr("origin_target_name")?.extract()?; + write_line( + &invocation.stream, + &render_clone_header( + (&origin, &invocation.destination), + destination_entry_count(&pipeline)?, + invocation.style, + ), + )?; + let (result, elapsed) = execute_clone_plan(request, invocation, (&pipeline, connection))?; + Ok((pipeline, result, elapsed)) +} + +/// Run one `sqb clone` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_clone_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let invocation: Invocation<'_> = invocation(py, request)?; + let (config, connection) = connect(request, &invocation)?; + let outcome = clone(request, &invocation, (&config, &connection)); + let closed: PyResult> = + invocation.adapter.call_method1("close", (&connection,)); + let (pipeline, result, elapsed) = outcome?; + closed?; + let items: Vec = result + .getattr("item_results")? + .try_iter()? + .map(|item| clone_item_text(&item?)) + .collect::>()?; + let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + for line in render_clone_completion(&items, Some(elapsed), invocation.style).lines() { + if !line.is_empty() { + stdout.call_method1("write", (line,))?; + } + stdout.call_method1("write", ("\n",))?; + } + let resource_types: Bound<'_, PyDict> = PyDict::new(py); + for (name, resource_type) in resource_types_by_name(&pipeline)? { + resource_types.set_item(name, resource_type)?; + } + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_clone_execution_json")?, + &[ + ("result", result.clone()), + ("resource_types_by_name", resource_types.into_any()), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", PyBool::new(py, false).to_owned().into_any()), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(clone_exit_code(&items)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_clone_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs new file mode 100644 index 0000000000..3bc319110d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -0,0 +1,382 @@ +//! Execute the prepared clone plan and propagate fingerprints. + +use std::collections::HashMap; +use std::time::Instant; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString}; +use sqlbuild_core::clone_text::main::clone_relation_flow_text::clone_relation_flow_text; +use sqlbuild_core::clone_text::models::FingerprintProgress; + +use crate::bindings::_helpers::clone::clone_command::Invocation; +use crate::bindings::_helpers::clone::clone_progress::{ + CloneFingerprintReporter, CloneItemPrinter, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; +const CLONE_PACKAGE: &str = "sqlbuild.executor.clone.main"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ + "destination_source_entries", + "destination_model_entries", + "destination_seed_entries", + "destination_function_entries", +]; +const ORIGIN_ENTRY_GROUPS: [&str; 3] = [ + "origin_source_entries", + "origin_model_entries", + "origin_seed_entries", +]; + +/// The pipeline's plan entries of `groups`, in order. +fn entries<'py>(pipeline: &Bound<'py, PyAny>, groups: &[&str]) -> PyResult>> { + let mut found: Vec> = Vec::new(); + for group in groups { + for entry in pipeline.getattr(*group)?.try_iter()? { + found.push(entry?); + } + } + Ok(found) +} + +/// The number of destination entries, which the header counts. +pub(crate) fn destination_entry_count(pipeline: &Bound<'_, PyAny>) -> PyResult { + Ok(entries(pipeline, &DESTINATION_ENTRY_GROUPS)?.len()) +} + +/// Each destination entry's resource type by name. +pub(crate) fn resource_types_by_name( + pipeline: &Bound<'_, PyAny>, +) -> PyResult> { + entries(pipeline, &DESTINATION_ENTRY_GROUPS)? + .iter() + .map(|entry| { + Ok(( + entry.getattr("name")?.extract()?, + entry + .getattr("key")? + .getattr("resource_type")? + .str()? + .extract()?, + )) + }) + .collect() +} + +fn relation_text(entry: &Bound<'_, PyAny>) -> PyResult> { + let destination: Bound<'_, PyAny> = entry.getattr("destination")?; + let qualified: Bound<'_, PyAny> = destination.getattr("qualified_name")?; + let relation: Bound<'_, PyAny> = if qualified.is_truthy()? { + qualified + } else { + destination.getattr("name")? + }; + if relation.is_none() { + return Ok(None); + } + relation.extract().map(Some) +} + +/// The widest plain relation flow among the destination entries. +fn relation_width(pipeline: &Bound<'_, PyAny>) -> PyResult { + let mut origins: HashMap> = HashMap::new(); + for entry in entries(pipeline, &ORIGIN_ENTRY_GROUPS)? { + origins.insert(entry.getattr("name")?.extract()?, relation_text(&entry)?); + } + let mut width: usize = 0; + for entry in entries(pipeline, &DESTINATION_ENTRY_GROUPS)? { + let name: String = entry.getattr("name")?.extract()?; + let destination: Option = relation_text(&entry)?; + let origin: Option<&str> = origins.get(&name).and_then(Option::as_deref); + let flow: String = clone_relation_flow_text(&name, (origin, destination.as_deref())); + width = width.max(flow.chars().count()); + } + Ok(width) +} + +/// Destination relation locations of every planned entry and of each source table. +fn dependency_locations<'py>( + pipeline: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = pipeline.py(); + let locations: Bound<'py, PyDict> = PyDict::new(py); + for entry in entries(pipeline, &DESTINATION_ENTRY_GROUPS)? { + locations.set_item(entry.getattr("key")?, entry.getattr("destination")?)?; + } + let plan: Bound<'py, PyAny> = pipeline.getattr("clone_plan")?; + let read_map: Bound<'py, PyAny> = plan.getattr("source_read_map")?; + let source_map: Bound<'py, PyAny> = if read_map.is_truthy()? { + read_map + } else { + plan.getattr("source_map")? + }; + let key_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS, "CompiledObjectKey")?; + let location_class: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS, "CompiledRelationLocation")?; + let source_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + for pair in source_map.call_method0("items")?.try_iter()? { + let (name, source): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + let table: Bound<'py, PyAny> = source.getattr("table")?; + if table.is_none() { + continue; + } + let place = [ + ("database", source.getattr("database")?), + ("schema", source.getattr("schema")?), + ("name", table.clone()), + ]; + let rendered: Bound<'py, PyAny> = + call_with(&adapter.getattr("render_qualified_name")?, &place)?; + let qualified: Bound<'py, PyAny> = if rendered.is_truthy()? { + rendered + } else { + table + }; + let [database, schema, table] = place; + let location: Bound<'py, PyAny> = call_with( + &location_class, + &[database, schema, table, ("qualified_name", qualified)], + )?; + locations.set_item(key_class.call1((&source_type, name))?, location)?; + } + Ok(locations) +} + +fn target_config<'py>(invocation: &Invocation<'py>) -> PyResult> { + let py: Python<'py> = invocation.adapter.py(); + call_with( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ( + "project_config", + invocation.discovered.getattr("project_config")?, + ), + ( + "local_config", + invocation.discovered.getattr("local_config")?, + ), + ( + "target_name", + PyString::new(py, &invocation.destination).into_any(), + ), + ], + ) +} + +fn clone_function<'py>( + py: Python<'py>, + (module, name): (&str, &str), +) -> PyResult> { + imported(py, &format!("{CLONE_PACKAGE}.{module}"), name) +} + +/// The executor input for the whole clone, streaming items through `on_item`. +fn execution_input<'py>( + invocation: &Invocation<'py>, + (pipeline, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (hard_copy, on_item): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = pipeline.py(); + let project: Bound<'py, PyAny> = pipeline.getattr("destination_project")?; + let plan: Bound<'py, PyAny> = pipeline.getattr("clone_plan")?; + let config: Bound<'py, PyAny> = target_config(invocation)?; + let selected: Vec> = pipeline + .getattr("destination_model_entries")? + .try_iter()? + .map(|entry| entry?.getattr("name")) + .collect::>()?; + let retention: Bound<'py, PyAny> = call_with( + &clone_function( + py, + ( + "build_retention_requests", + "build_destination_retention_requests", + ), + )?, + &[ + ("project", project.clone()), + ( + "adapter_name", + PyString::new(py, &invocation.adapter_name).into_any(), + ), + ( + "selected_model_names", + PyFrozenSet::new(py, selected)?.into_any(), + ), + ( + "namespace_owned", + config.getattr("owns_time_travel_retention_namespace")?, + ), + ], + )?; + let allow_decrease: bool = config + .getattr("time_travel_retention_decrease")? + .eq(imported(py, CONTRACT_TYPES, "RetentionDecreasePolicy")?.getattr("ALLOW")?)?; + let source_entries: Bound<'py, PyAny> = call_with( + &imported(py, CLONE_MODELS, "CloneSourceEntries")?, + &[ + ("origin", pipeline.getattr("origin_source_entries")?), + ( + "destination", + pipeline.getattr("destination_source_entries")?, + ), + ], + )?; + let transient: Bound<'py, PyAny> = call_with( + &clone_function( + py, + ( + "build_destination_transient_models", + "build_destination_transient_models", + ), + )?, + &[("project", project.clone())], + )?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![("source_entries", source_entries)]; + for group in [ + "origin_model_entries", + "destination_model_entries", + "origin_seed_entries", + "destination_seed_entries", + "destination_function_entries", + ] { + fields.push((group, pipeline.getattr(group)?)); + } + fields.extend([ + ("execution_order", plan.getattr("execution_order")?), + ("adapter", invocation.adapter.clone()), + ("destination_connection", connection.clone()), + ("hard_copy", hard_copy), + ("run_id", project.getattr("run_id")?), + ( + "query_change_tracking", + project + .getattr("settings")? + .getattr("query_change_tracking")?, + ), + ("upstream_deps", plan.getattr("upstream_deps")?), + ( + "dependency_locations", + dependency_locations(pipeline, &invocation.adapter)?.into_any(), + ), + ("destination_transient_models", transient), + ("destination_retention_requests", retention), + ( + "allow_namespace_retention_decrease", + PyBool::new(py, allow_decrease).to_owned().into_any(), + ), + ("on_item", on_item), + ]); + call_with(&imported(py, CLONE_MODELS, "CloneExecutionInput")?, &fields) +} + +/// Copy fingerprints of the cloned relations, reporting what was not confirmed on failure. +fn copy_fingerprints<'py>( + invocation: &Invocation<'py>, + (pipeline, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + result: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = pipeline.py(); + let reporter: Bound<'py, CloneFingerprintReporter> = Bound::new( + py, + CloneFingerprintReporter { + stream: invocation.stream.clone().unbind(), + style: invocation.style, + progress: FingerprintProgress::default(), + }, + )?; + let project: Bound<'py, PyAny> = pipeline.getattr("destination_project")?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![("result", result.clone())]; + for group in [ + "origin_model_entries", + "destination_model_entries", + "origin_seed_entries", + "destination_seed_entries", + ] { + fields.push((group, pipeline.getattr(group)?)); + } + fields.extend([ + ("adapter", invocation.adapter.clone()), + ("destination_connection", connection.clone()), + ("run_id", project.getattr("run_id")?), + ( + "query_change_tracking", + project + .getattr("settings")? + .getattr("query_change_tracking")?, + ), + ("on_progress", reporter.clone().into_any()), + ]); + let lifecycle: Bound<'py, PyAny> = call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", PyString::new(py, "clone").into_any()), + ( + "operation_name", + PyString::new(py, "clone_finalization").into_any(), + ), + ], + )?; + let copied: PyResult> = within(&lifecycle, |_| { + call_with( + &clone_function(py, ("fingerprinting", "copy_clone_fingerprints"))?, + &fields, + ) + }); + if let Err(error) = copied { + reporter.borrow().write_interrupted(py)?; + return Err(error); + } + Ok(()) +} + +/// Execute the clone, streaming items, then copy fingerprints; returns the result and elapsed seconds. +pub(crate) fn execute_clone_plan<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, + (pipeline, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyAny>, f64)> { + let py: Python<'py> = pipeline.py(); + let started: Instant = Instant::now(); + let event_writer: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ExecutionEventWriter")?, + &[("path", request.getattr("event_output_path")?)], + )?; + let printer: Py = Py::new( + py, + CloneItemPrinter { + stream: invocation.stream.clone().unbind(), + style: invocation.style, + relation_width: relation_width(pipeline)?, + event_writer: event_writer.clone().unbind(), + resource_types: resource_types_by_name(pipeline)?, + }, + )?; + let executed: PyResult> = (|| { + let input: Bound<'py, PyAny> = execution_input( + invocation, + (pipeline, connection), + ( + request.getattr("hard_copy")?, + printer.into_bound(py).into_any(), + ), + )?; + call_with( + &clone_function(py, ("execute", "execute_clone"))?, + &[("inputs", input)], + ) + })(); + let closed: PyResult> = event_writer.call_method0("close"); + let result: Bound<'py, PyAny> = executed?; + closed?; + copy_fingerprints(invocation, (pipeline, connection), &result)?; + Ok((result, started.elapsed().as_secs_f64())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs new file mode 100644 index 0000000000..15a3ebb780 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs @@ -0,0 +1,129 @@ +//! Streamed clone item lines and fingerprint propagation progress, as executor callbacks. + +use std::collections::HashMap; + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::clone_text::main::render_clone_fingerprint_interrupted_line::render_clone_fingerprint_interrupted_line; +use sqlbuild_core::clone_text::main::render_clone_fingerprint_progress_line::render_clone_fingerprint_progress_line; +use sqlbuild_core::clone_text::main::render_clone_item_line::render_clone_item_line; +use sqlbuild_core::clone_text::models::{CloneItemText, CloneStatusKind, FingerprintProgress}; + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +/// One executor `CloneItemResult` as the command renders it. +pub(crate) fn clone_item_text(item: &Bound<'_, PyAny>) -> PyResult { + let duration: Bound<'_, PyAny> = item.getattr("duration_seconds")?; + Ok(CloneItemText { + name: item.getattr("name")?.extract()?, + action: item.getattr("action")?.getattr("value")?.extract()?, + status: CloneStatusKind::from_value( + &item + .getattr("status")? + .getattr("value")? + .extract::()?, + ), + message: optional_text(item.getattr("message")?)?, + origin_relation: optional_text(item.getattr("origin_relation")?)?, + destination_relation: optional_text(item.getattr("destination_relation")?)?, + duration_seconds: if duration.is_none() { + None + } else { + Some(duration.extract()?) + }, + }) +} + +/// Write `line` and a newline to `stream` and flush. +pub(crate) fn write_line(stream: &Bound<'_, PyAny>, line: &str) -> PyResult<()> { + stream.call_method1("write", (format!("{line}\n"),))?; + stream.call_method0("flush")?; + Ok(()) +} + +/// Print each finished clone item and record it in the event output. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CloneItemPrinter { + pub(crate) stream: Py, + pub(crate) style: CliStyle, + pub(crate) relation_width: usize, + pub(crate) event_writer: Py, + pub(crate) resource_types: HashMap, +} + +#[pymethods] +impl CloneItemPrinter { + #[pyo3(signature = (*, index, total, item))] + fn __call__( + &self, + py: Python<'_>, + index: usize, + total: usize, + item: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let text: CloneItemText = clone_item_text(item)?; + write_line( + self.stream.bind(py), + &render_clone_item_line((index, total), &text, self.relation_width, self.style), + )?; + let resource_type: &String = self + .resource_types + .get(&text.name) + .ok_or_else(|| PyKeyError::new_err(text.name.clone()))?; + let options: Bound<'_, PyDict> = PyDict::new(py); + pyo3::types::PyDictMethods::set_item(&options, "item", item)?; + pyo3::types::PyDictMethods::set_item(&options, "resource_type", resource_type)?; + self.event_writer + .bind(py) + .call_method("write_clone_result", (), Some(&options))?; + Ok(()) + } +} + +/// Report confirmed fingerprint batches and, on interruption, what was not confirmed. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct CloneFingerprintReporter { + pub(crate) stream: Py, + pub(crate) style: CliStyle, + pub(crate) progress: FingerprintProgress, +} + +#[pymethods] +impl CloneFingerprintReporter { + #[pyo3(signature = (*, completed, total, pending_identities))] + fn __call__( + &mut self, + py: Python<'_>, + completed: i64, + total: i64, + pending_identities: Vec, + ) -> PyResult<()> { + self.progress = FingerprintProgress { + completed, + total, + pending_identities, + }; + if completed == 0 { + return Ok(()); + } + write_line( + self.stream.bind(py), + &render_clone_fingerprint_progress_line(completed, total, self.style), + ) + } + + pub(crate) fn write_interrupted(&self, py: Python<'_>) -> PyResult<()> { + match render_clone_fingerprint_interrupted_line(&self.progress, self.style) { + Some(line) => write_line(self.stream.bind(py), &line), + None => Ok(()), + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs new file mode 100644 index 0000000000..6ac5b67d26 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod clone_command; +pub(crate) mod clone_execution; +pub(crate) mod clone_progress; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs new file mode 100644 index 0000000000..9df257097e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs @@ -0,0 +1,9 @@ +//! Register the native `sqb clone` command. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::clone::clone_command; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + clone_command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 04923fcfd4..ecd35332b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod authored_values; pub(crate) mod boundary; pub(crate) mod cache; pub(crate) mod cli; +pub(crate) mod clone; pub(crate) mod command_runtime; pub(crate) mod compile_attachment; pub(crate) mod compile_command; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 70d923473f..e42c8d671b 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,7 +5,7 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, - command_runtime, compile_attachment, compile_command, compile_driver, compile_outputs, + clone, command_runtime, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cost_report, cursor, dag_artifact, debug_command, discovery, fingerprints, freshness, frontier, graph, janitor, lineage, macro_bridge, manifest, model_config, plan_output, presentation_text, project_assembly, refactor, references, render, @@ -47,6 +47,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { refactor::registration::register(module)?; janitor::registration::register(module)?; freshness::registration::register(module)?; + clone::registration::register(module)?; plan_output::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b92c6231d8..f37f91ff5d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -18,6 +18,7 @@ from sqlbuild.adapter.contract.types import ( ) from sqlbuild.adapter.relations.models import InspectionQueryRecord from sqlbuild.cli.commands.models import ( + CloneCommandRequest, FreshnessCommandRequest, JanitorCommandRequest, LineageCommandRequest, @@ -2737,6 +2738,7 @@ def run_lineage_command(*, request: LineageCommandRequest) -> int: ... def run_refactor_command(*, request: RefactorCommandRequest) -> int: ... def run_janitor_command(*, request: JanitorCommandRequest) -> int: ... def run_freshness_command(*, request: FreshnessCommandRequest) -> int: ... +def run_clone_command(*, request: CloneCommandRequest) -> int: ... def observe_source_freshness_json( *, adapter: object, diff --git a/src/sqlbuild/cli/commands/_helpers/clone/__init__.py b/src/sqlbuild/cli/commands/_helpers/clone/__init__.py deleted file mode 100644 index ea2285a6d1..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Clone command helper package.""" diff --git a/src/sqlbuild/cli/commands/_helpers/clone/connections.py b/src/sqlbuild/cli/commands/_helpers/clone/connections.py deleted file mode 100644 index 0f6a691b29..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/connections.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Clone command connection phases.""" - -from __future__ import annotations - -import time -from typing import Any - -from sqlbuild._native import ( - PlanningProgressReporter, - resolve_target_connection_config, -) -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneConnectionContext, - CloneInvocation, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.phase_line import format_phase_line -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def connect_clone_targets( - *, request: CloneCommandRequest, invocation: CloneInvocation -) -> CloneConnectionContext: - """Resolve and connect the destination target used for the whole clone.""" - - destination_connection_config: dict[str, object] = resolve_target_connection_config( - discovered_inputs=invocation.discovered_inputs, - project_dir=invocation.effective_project_dir, - target_name=invocation.destination_target_name, - cli_vars=request.cli_vars, - ) - progress: PlanningProgressReporter = PlanningProgressReporter( - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - progress.start(f"Connecting to {invocation.adapter_name}...") - connect_start: float = time.monotonic() - with OperationLifecycle(operation_kind="clone", operation_name="clone_target_connection"): - destination_connection: Any = invocation.adapter.connect(destination_connection_config) - progress.complete_styled( - format_phase_line( - style=CliStyle(use_color=invocation.use_color), - ok=True, - label="Warehouse connected", - summary=(f"{invocation.adapter_name} ({time.monotonic() - connect_start:.2f}s)"), - ) - ) - return CloneConnectionContext( - destination_connection_config=destination_connection_config, - destination_connection=destination_connection, - ) - - -def close_clone_targets( - *, invocation: CloneInvocation, connection_context: CloneConnectionContext -) -> None: - """Close the clone destination connection.""" - - invocation.adapter.close(connection_context.destination_connection) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/execution.py b/src/sqlbuild/cli/commands/_helpers/clone/execution.py deleted file mode 100644 index b8316bb9ce..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/execution.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Clone command execution phases.""" - -from __future__ import annotations - -import time -from typing import TextIO - -from sqlbuild._native import ExecutionEventWriter -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.clone.fingerprint_progress import ( - create_clone_fingerprint_progress_reporter, -) -from sqlbuild.cli.commands._helpers.clone.output import ( - clone_relation_flow_text, - render_clone_item_line, -) -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneConnectionContext, - CloneExecutionPreparation, - CloneInvocation, - CloneRunOutcome, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - FunctionPlanEntry, - ModelPlanEntry, - SeedPlanEntry, -) -from sqlbuild.executor.clone.main.build_destination_transient_models import ( - build_destination_transient_models, -) -from sqlbuild.executor.clone.main.build_retention_requests import ( - build_destination_retention_requests, -) -from sqlbuild.executor.clone.main.execute import execute_clone -from sqlbuild.executor.clone.main.fingerprinting import copy_clone_fingerprints -from sqlbuild.executor.clone.models import ( - CloneExecutionInput, - CloneExecutionResult, - CloneItemResult, - CloneSourceEntries, -) -from sqlbuild.executor.clone.types import CloneFingerprintProgressReporter, CloneItemCallback -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import RetentionDecreasePolicy - - -def execute_clone_plan( - *, - request: CloneCommandRequest, - invocation: CloneInvocation, - connection_context: CloneConnectionContext, - preparation: CloneExecutionPreparation, -) -> CloneRunOutcome: - """Execute the direct clone plan and copy fingerprints.""" - - clone_start: float = time.monotonic() - resource_types_by_name: dict[str, str] = _clone_resource_types_by_name(preparation=preparation) - event_writer: ExecutionEventWriter = ExecutionEventWriter(path=request.event_output_path) - relation_width: int = _clone_relation_width(preparation=preparation) - on_item: CloneItemCallback = _build_on_clone_item( - stream=invocation.progress_stream, - use_color=invocation.use_color, - event_writer=event_writer, - resource_types_by_name=resource_types_by_name, - relation_width=relation_width, - ) - try: - result: CloneExecutionResult = execute_clone( - inputs=CloneExecutionInput( - source_entries=CloneSourceEntries( - origin=preparation.pipeline_result.origin_source_entries, - destination=preparation.pipeline_result.destination_source_entries, - ), - origin_model_entries=preparation.pipeline_result.origin_model_entries, - destination_model_entries=preparation.pipeline_result.destination_model_entries, - origin_seed_entries=preparation.pipeline_result.origin_seed_entries, - destination_seed_entries=preparation.pipeline_result.destination_seed_entries, - destination_function_entries=preparation.pipeline_result.destination_function_entries, - execution_order=preparation.pipeline_result.clone_plan.execution_order, - adapter=invocation.adapter, - destination_connection=connection_context.destination_connection, - hard_copy=request.hard_copy, - run_id=preparation.pipeline_result.destination_project.run_id, - query_change_tracking=preparation.pipeline_result.destination_project.settings.query_change_tracking, - upstream_deps=preparation.pipeline_result.clone_plan.upstream_deps, - dependency_locations=_clone_dependency_locations( - preparation=preparation, - adapter=invocation.adapter, - ), - destination_transient_models=build_destination_transient_models( - project=preparation.pipeline_result.destination_project - ), - destination_retention_requests=build_destination_retention_requests( - project=preparation.pipeline_result.destination_project, - adapter_name=invocation.adapter_name, - selected_model_names=frozenset( - entry.name - for entry in preparation.pipeline_result.destination_model_entries - ), - namespace_owned=resolve_target_config( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - target_name=invocation.destination_target_name, - ).owns_time_travel_retention_namespace, - ), - allow_namespace_retention_decrease=( - resolve_target_config( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - target_name=invocation.destination_target_name, - ).time_travel_retention_decrease - == RetentionDecreasePolicy.ALLOW - ), - on_item=on_item, - ) - ) - finally: - event_writer.close() - fingerprint_progress: CloneFingerprintProgressReporter = ( - create_clone_fingerprint_progress_reporter( - stream=invocation.progress_stream, use_color=invocation.use_color - ) - ) - try: - with OperationLifecycle(operation_kind="clone", operation_name="clone_finalization"): - copy_clone_fingerprints( - result=result, - origin_model_entries=preparation.pipeline_result.origin_model_entries, - destination_model_entries=preparation.pipeline_result.destination_model_entries, - origin_seed_entries=preparation.pipeline_result.origin_seed_entries, - destination_seed_entries=preparation.pipeline_result.destination_seed_entries, - adapter=invocation.adapter, - destination_connection=connection_context.destination_connection, - run_id=preparation.pipeline_result.destination_project.run_id, - query_change_tracking=preparation.pipeline_result.destination_project.settings.query_change_tracking, - on_progress=fingerprint_progress, - ) - except BaseException: - fingerprint_progress.write_interrupted() - raise - return CloneRunOutcome(result=result, elapsed=time.monotonic() - clone_start) - - -def _clone_dependency_locations( - *, preparation: CloneExecutionPreparation, adapter: BaseAdapter -) -> dict[CompiledObjectKey, CompiledRelationLocation]: - entries: tuple[ - CloneSourcePlanEntry | ModelPlanEntry | SeedPlanEntry | FunctionPlanEntry, ... - ] = ( - *preparation.pipeline_result.destination_source_entries, - *preparation.pipeline_result.destination_model_entries, - *preparation.pipeline_result.destination_seed_entries, - *preparation.pipeline_result.destination_function_entries, - ) - locations: dict[CompiledObjectKey, CompiledRelationLocation] = { - entry.key: entry.destination for entry in entries - } - source_map: dict[str, SourceEntry] = ( - preparation.pipeline_result.clone_plan.source_read_map - or preparation.pipeline_result.clone_plan.source_map - ) - for name, source in source_map.items(): - if source.table is None: - continue - key: CompiledObjectKey = CompiledObjectKey(CompiledResourceType.SOURCE, name) - locations[key] = CompiledRelationLocation( - database=source.database, - schema=source.schema, - name=source.table, - qualified_name=( - adapter.render_qualified_name( - database=source.database, - schema=source.schema, - name=source.table, - ) - or source.table - ), - ) - return locations - - -def _clone_resource_types_by_name(*, preparation: CloneExecutionPreparation) -> dict[str, str]: - entries: tuple[ - CloneSourcePlanEntry | ModelPlanEntry | SeedPlanEntry | FunctionPlanEntry, ... - ] = ( - *preparation.pipeline_result.destination_source_entries, - *preparation.pipeline_result.destination_model_entries, - *preparation.pipeline_result.destination_seed_entries, - *preparation.pipeline_result.destination_function_entries, - ) - return {entry.name: str(entry.key.resource_type) for entry in entries} - - -def _build_on_clone_item( - *, - stream: TextIO, - use_color: bool, - event_writer: ExecutionEventWriter, - resource_types_by_name: dict[str, str], - relation_width: int, -) -> CloneItemCallback: - def _on_clone_item(*, index: int, total: int, item: CloneItemResult) -> None: - stream.write( - render_clone_item_line( - index=index, - total=total, - item=item, - use_color=use_color, - relation_width=relation_width, - ) - + "\n" - ) - stream.flush() - event_writer.write_clone_result( - item=item, - resource_type=resource_types_by_name[item.name], - ) - - return _on_clone_item - - -def _clone_relation_width(*, preparation: CloneExecutionPreparation) -> int: - origin_entries: tuple[CloneSourcePlanEntry | ModelPlanEntry | SeedPlanEntry, ...] = ( - *preparation.pipeline_result.origin_source_entries, - *preparation.pipeline_result.origin_model_entries, - *preparation.pipeline_result.origin_seed_entries, - ) - origin_relations: dict[str, str | None] = { - entry.name: entry.destination.qualified_name or entry.destination.name - for entry in origin_entries - } - destination_entries: tuple[ - CloneSourcePlanEntry | ModelPlanEntry | SeedPlanEntry | FunctionPlanEntry, ... - ] = ( - *preparation.pipeline_result.destination_source_entries, - *preparation.pipeline_result.destination_model_entries, - *preparation.pipeline_result.destination_seed_entries, - *preparation.pipeline_result.destination_function_entries, - ) - flows: tuple[str, ...] = tuple( - clone_relation_flow_text( - name=entry.name, - origin_relation=origin_relations.get(entry.name), - destination_relation=entry.destination.qualified_name or entry.destination.name, - ) - for entry in destination_entries - ) - return max((len(flow) for flow in flows), default=0) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/fingerprint_progress.py b/src/sqlbuild/cli/commands/_helpers/clone/fingerprint_progress.py deleted file mode 100644 index 39b77edcf6..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/fingerprint_progress.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Clone fingerprint finalization progress reporting.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.cli.commands._helpers.clone.output import ( - render_clone_fingerprint_interrupted_line, - render_clone_fingerprint_progress_line, -) -from sqlbuild.executor.clone.types import CloneFingerprintProgressReporter - - -class _CloneFingerprintProgressReporter: - def __init__(self, *, stream: TextIO, use_color: bool) -> None: - self._stream: TextIO = stream - self._use_color: bool = use_color - self._confirmed: int = 0 - self._total: int = 0 - self._pending_identities: tuple[str, ...] = () - - def __call__(self, *, completed: int, total: int, pending_identities: tuple[str, ...]) -> None: - self._confirmed = completed - self._total = total - self._pending_identities = pending_identities - if completed == 0: - return - self._stream.write( - render_clone_fingerprint_progress_line( - completed=completed, - total=total, - use_color=self._use_color, - ) - + "\n" - ) - self._stream.flush() - - def write_interrupted(self) -> None: - if self._total <= self._confirmed: - return - self._stream.write( - render_clone_fingerprint_interrupted_line( - completed=self._confirmed, - total=self._total, - pending_identities=self._pending_identities, - use_color=self._use_color, - ) - + "\n" - ) - self._stream.flush() - - -def create_clone_fingerprint_progress_reporter( - *, stream: TextIO, use_color: bool -) -> CloneFingerprintProgressReporter: - """Create a stateful reporter for one clone finalization phase.""" - - return _CloneFingerprintProgressReporter(stream=stream, use_color=use_color) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/invocation.py b/src/sqlbuild/cli/commands/_helpers/clone/invocation.py deleted file mode 100644 index c1a07cb339..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/invocation.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Clone command invocation resolution phase.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import PlanningProgressReporter, discover_project_inputs -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.clone.validation import validate_clone_request -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import CloneCommandRequest, CloneInvocation -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name - - -def resolve_clone_invocation(*, request: CloneCommandRequest) -> CloneInvocation: - """Resolve discovery, adapter, and output context for clone.""" - - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - use_color: bool = not request.no_color and supports_color() - progress_stream: TextIO = sys.stdout - progress: PlanningProgressReporter = PlanningProgressReporter( - stream=progress_stream, - use_color=use_color, - ) - progress.start("Discovering project...") - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - progress.complete("Discovered project.") - reject_unit_test_selectors( - select=request.select, exclude=request.exclude, discovered_inputs=discovered_inputs - ) - destination_target_name: str | None = request.destination_target_name - if destination_target_name is None: - destination_target_name = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - if destination_target_name is None: - raise CliUserError( - "clone requires --to when the project has no active target", - code="C406", - ) - validate_clone_request( - discovered_inputs=discovered_inputs, - origin_target_name=request.origin_target_name, - destination_target_name=destination_target_name, - ) - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, project_dir=effective_project_dir - ) - return CloneInvocation( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - adapter_name=adapter_name, - adapter=adapter, - destination_target_name=destination_target_name, - use_color=use_color, - progress_stream=progress_stream, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/output.py b/src/sqlbuild/cli/commands/_helpers/clone/output.py deleted file mode 100644 index 3d0b80a50b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/output.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Clone command output helpers.""" - -from __future__ import annotations - -import re - -from sqlbuild._native import format_completion_line, format_summary_footer -from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult -from sqlbuild.executor.clone.types import CloneAction, CloneStatus -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.types import CompletionState - -_QUOTED_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile( - r'("(?:[^"]|"")*"|`(?:[^`]|``)*`|\[(?:[^\]]|\]\])*\])' -) - - -def render_clone_header( - *, origin_target_name: str, destination_target_name: str, total: int, use_color: bool -) -> str: - """Render the persistent clone header naming origin, destination, and resource count.""" - - style: CliStyle = CliStyle(use_color=use_color) - return ( - f"{style.title('sqb clone')} " - f"{style.label('origin=')}{style.value(origin_target_name)} " - f"{style.label('destination=')}{style.value(destination_target_name)} " - f"{style.muted(f'({total} resource{"" if total == 1 else "s"})')}" - ) - - -def _clone_status_text(status: CloneStatus) -> str: - if status == CloneStatus.WARNING: - return "WARN" - if status == CloneStatus.FAILED: - return "FAIL" - return "OK" - - -def clone_relation_flow_text( - *, name: str, origin_relation: str | None, destination_relation: str | None -) -> str: - """Return the normalized plain-text relation flow used for clone progress.""" - - if origin_relation is None or destination_relation is None: - return name.lower() - return ( - f"{clone_relation_display_name(origin_relation)} -> " - f"{clone_relation_display_name(destination_relation)}" - ) - - -def render_clone_fingerprint_progress_line(*, completed: int, total: int, use_color: bool) -> str: - """Render confirmed clone fingerprint propagation progress.""" - - style: CliStyle = CliStyle(use_color=use_color) - width: int = len(str(total)) - return ( - f" {'fingerprint':<10}" - f"[{completed:>{width}}/{total}] propagated {style.status(status='OK')}" - ) - - -def render_clone_fingerprint_interrupted_line( - *, completed: int, total: int, pending_identities: tuple[str, ...], use_color: bool -) -> str: - """Render bounded context for interrupted clone fingerprint propagation.""" - - style: CliStyle = CliStyle(use_color=use_color) - displayed: tuple[str, ...] = pending_identities[:10] - remaining_count: int = len(pending_identities) - len(displayed) - pending_text: str = ", ".join(displayed) - if remaining_count: - pending_text = f"{pending_text} (+{remaining_count} more)" - return ( - f" {'fingerprint':<10}" - f"[{completed}/{total}] {style.status(status='FAIL')} " - f"not confirmed: {pending_text}" - ) - - -def clone_relation_display_name(relation: str) -> str: - """Normalize ordinary identifiers while preserving quoted identifier case.""" - - parts: list[str] = _QUOTED_IDENTIFIER_PATTERN.split(relation) - return "".join( - part if _QUOTED_IDENTIFIER_PATTERN.fullmatch(part) else part.lower() for part in parts - ) - - -def render_clone_item_line( - *, - index: int, - total: int, - item: CloneItemResult, - use_color: bool, - relation_width: int | None = None, -) -> str: - """Render one streamed clone line: position, action, origin to destination, and status.""" - - style: CliStyle = CliStyle(use_color=use_color) - position: str = f"{index:>{len(str(total))}}/{total}" - plain_relation_flow: str = clone_relation_flow_text( - name=item.name, - origin_relation=item.origin_relation, - destination_relation=item.destination_relation, - ) - relation_flow: str = style.object_name(plain_relation_flow) - if item.origin_relation is not None and item.destination_relation is not None: - relation_flow = ( - f"{style.object_name(clone_relation_display_name(item.origin_relation))} " - f"{style.muted('->')} " - f"{style.object_name(clone_relation_display_name(item.destination_relation))}" - ) - effective_relation_width: int = max(len(plain_relation_flow), relation_width or 0) - relation_padding: str = " " * (effective_relation_width - len(plain_relation_flow)) - action: str = item.action.value - action_width: int = max(len(value.value) for value in CloneAction) - action_padding: str = " " * (action_width - len(action)) - duration: str = ( - f" {style.muted(f'{item.duration_seconds:.2f}s')}" - if item.duration_seconds is not None - else "" - ) - return ( - f" {style.muted(position)} {style.accent(action)}{action_padding} " - f"{relation_flow}{relation_padding} " - f"{style.status(status=_clone_status_text(status=item.status))}{duration}" - ) - - -def render_clone_output( - *, - result: CloneExecutionResult, - elapsed_seconds: float | None = None, - use_color: bool, -) -> None: - style: CliStyle = CliStyle(use_color=use_color) - item: CloneItemResult - for item in result.item_results: - if item.message is not None and item.status != CloneStatus.SUCCESS: - print(f" {style.object_name(item.name)} {style.muted(item.message)}") - success_count: int = sum( - 1 for item in result.item_results if item.status == CloneStatus.SUCCESS - ) - warning_count: int = sum( - 1 for item in result.item_results if item.status == CloneStatus.WARNING - ) - fail_count: int = sum(1 for item in result.item_results if item.status == CloneStatus.FAILED) - cloned_count: int = sum(1 for item in result.item_results if item.action == CloneAction.CLONED) - copied_count: int = sum(1 for item in result.item_results if item.action == CloneAction.COPIED) - recreated_count: int = sum( - 1 for item in result.item_results if item.action == CloneAction.RECREATED_VIEW - ) - recreated_function_count: int = sum( - 1 for item in result.item_results if item.action == CloneAction.RECREATED_FUNCTION - ) - print() - elapsed: str = f"({elapsed_seconds:.2f}s)" if elapsed_seconds is not None else "" - counts_summary: str = format_summary_footer( - counts=( - ("CLONED", cloned_count), - ("COPIED", copied_count), - ("RECREATED_VIEWS", recreated_count), - ("RECREATED_FUNCTIONS", recreated_function_count), - ("PASS", success_count), - ("WARN", warning_count), - ("FAIL", fail_count), - ("TOTAL", len(result.item_results)), - ), - use_color=use_color, - elapsed=elapsed or None, - ) - if warning_count == 0 and fail_count == 0: - completion_state: CompletionState = CompletionState.OK - completion_label: str = "Completed successfully" - elif fail_count == 0: - completion_state = CompletionState.WARN - completion_label = "Completed with warnings" - else: - completion_state = CompletionState.FAIL - completion_label = "Completed with errors" - print( - format_completion_line( - style=style, - state=completion_state, - label=completion_label, - summary=counts_summary, - ) - ) - - -def is_clone_success(result: CloneExecutionResult) -> bool: - return all(item.status != CloneStatus.FAILED for item in result.item_results) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/outputs.py b/src/sqlbuild/cli/commands/_helpers/clone/outputs.py deleted file mode 100644 index 41adf45c2f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/outputs.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Clone command output phases.""" - -from __future__ import annotations - -from sqlbuild._native import format_clone_execution_json, write_execution_json_output -from sqlbuild.cli.commands._helpers.clone.output import ( - is_clone_success, - render_clone_header, - render_clone_output, -) -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneExecutionPreparation, - CloneInvocation, - CloneRunOutcome, -) -from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - FunctionPlanEntry, - ModelPlanEntry, - SeedPlanEntry, -) - - -def write_clone_execution_header( - *, - request: CloneCommandRequest, - invocation: CloneInvocation, - preparation: CloneExecutionPreparation, -) -> None: - """Write the direct clone execution header.""" - - clone_total: int = ( - len(preparation.pipeline_result.destination_source_entries) - + len(preparation.pipeline_result.destination_model_entries) - + len(preparation.pipeline_result.destination_seed_entries) - + len(preparation.pipeline_result.destination_function_entries) - ) - invocation.progress_stream.write( - render_clone_header( - origin_target_name=request.origin_target_name, - destination_target_name=invocation.destination_target_name, - total=clone_total, - use_color=invocation.use_color, - ) - + "\n" - ) - invocation.progress_stream.flush() - - -def write_clone_completion_output(*, invocation: CloneInvocation, outcome: CloneRunOutcome) -> None: - """Render final direct clone command output.""" - - render_clone_output( - result=outcome.result, - elapsed_seconds=outcome.elapsed, - use_color=invocation.use_color, - ) - - -def write_clone_execution_json_output( - *, - request: CloneCommandRequest, - preparation: CloneExecutionPreparation, - outcome: CloneRunOutcome, -) -> None: - """Write structured direct clone outcomes when requested.""" - - entries: tuple[ - CloneSourcePlanEntry | ModelPlanEntry | SeedPlanEntry | FunctionPlanEntry, ... - ] = ( - *preparation.pipeline_result.destination_source_entries, - *preparation.pipeline_result.destination_model_entries, - *preparation.pipeline_result.destination_seed_entries, - *preparation.pipeline_result.destination_function_entries, - ) - resource_types_by_name: dict[str, str] = { - entry.name: str(entry.key.resource_type) for entry in entries - } - write_execution_json_output( - payload=format_clone_execution_json( - result=outcome.result, - resource_types_by_name=resource_types_by_name, - ), - json_output=False, - json_output_path=request.json_output_path, - ) - - -def resolve_clone_exit_code(outcome: CloneRunOutcome) -> int: - """Return the shell exit code for a direct clone outcome.""" - - return 0 if is_clone_success(outcome.result) else 1 diff --git a/src/sqlbuild/cli/commands/_helpers/clone/planning.py b/src/sqlbuild/cli/commands/_helpers/clone/planning.py deleted file mode 100644 index 82b9b6d6c8..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/planning.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Clone command planning phase.""" - -from __future__ import annotations - -import time - -from sqlbuild._native import PlanningProgressReporter -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneConnectionContext, - CloneExecutionPreparation, - CloneInvocation, -) -from sqlbuild.compiler.pipeline.main.clone import run_clone_pipeline -from sqlbuild.compiler.pipeline.models import ClonePipelineConnection, ClonePipelineResult -from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - FunctionPlanEntry, - ModelPlanEntry, - SeedPlanEntry, -) -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def prepare_clone_execution( - *, - request: CloneCommandRequest, - invocation: CloneInvocation, - connection_context: CloneConnectionContext, -) -> CloneExecutionPreparation: - """Prepare the direct clone plan.""" - - progress: PlanningProgressReporter = PlanningProgressReporter( - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - planning_start: float = time.monotonic() - with OperationLifecycle(operation_kind="project", operation_name="project_compile"): - progress.on_progress("Preparing clone plan...") - pipeline_result: ClonePipelineResult = run_clone_pipeline( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - origin_target_name=request.origin_target_name, - destination_target_name=invocation.destination_target_name, - no_sql_validation=request.no_sql_validation, - select=request.select, - exclude=request.exclude, - cli_vars=request.cli_vars, - destination_connection=ClonePipelineConnection( - config=connection_context.destination_connection_config, - handle=connection_context.destination_connection, - ), - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ) - progress.on_progress(f"Prepared clone plan. ({time.monotonic() - planning_start:.2f}s)") - destination_model_entries: tuple[ModelPlanEntry, ...] = ( - pipeline_result.destination_model_entries - ) - destination_seed_entries: tuple[SeedPlanEntry, ...] = pipeline_result.destination_seed_entries - destination_source_entries: tuple[CloneSourcePlanEntry, ...] = ( - pipeline_result.destination_source_entries - ) - destination_function_entries: tuple[FunctionPlanEntry, ...] = ( - pipeline_result.destination_function_entries - ) - if ( - not destination_model_entries - and not destination_seed_entries - and not destination_source_entries - and not destination_function_entries - ): - raise CliUserError("no cloneable resources found in the selected scope", code="C407") - return CloneExecutionPreparation(pipeline_result=pipeline_result) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/validation.py b/src/sqlbuild/cli/commands/_helpers/clone/validation.py deleted file mode 100644 index 8c96cade8d..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/validation.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Clone command validation helpers.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.models import TargetConfig - - -def validate_clone_request( - *, - discovered_inputs: DiscoveredProjectInputs, - origin_target_name: str, - destination_target_name: str, -) -> None: - project_targets: dict[str, TargetConfig] = discovered_inputs.project_config.targets - local_target_names: set[str] = set(discovered_inputs.local_config.targets) - target_names: set[str] = set(project_targets) | local_target_names - if origin_target_name == destination_target_name: - raise CliUserError("clone requires different --from and --to targets", code="C401") - if origin_target_name not in target_names: - raise CliUserError(f"unknown target '{origin_target_name}'", code="C402") - if destination_target_name not in target_names: - raise CliUserError(f"unknown target '{destination_target_name}'", code="C403") - origin_target: TargetConfig = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=origin_target_name, - ) - destination_target: TargetConfig = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=destination_target_name, - ) - if not origin_target.clone.allow_as_clone_origin: - raise CliUserError( - ( - f"target '{origin_target_name}' is not allowed as a clone origin target; " - f"set targets.{origin_target_name}.clone.allow_as_clone_origin = true" - ), - code="C404", - ) - if not destination_target.clone.allow_as_clone_destination: - raise CliUserError( - ( - f"target '{destination_target_name}' is not allowed as a clone destination " - f"target; set targets.{destination_target_name}.clone." - "allow_as_clone_destination = true" - ), - code="C405", - ) diff --git a/src/sqlbuild/cli/commands/main/inspection/_clone.py b/src/sqlbuild/cli/commands/main/inspection/_clone.py deleted file mode 100644 index 996a76626b..0000000000 --- a/src/sqlbuild/cli/commands/main/inspection/_clone.py +++ /dev/null @@ -1,60 +0,0 @@ -"""CLI clone command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.clone.connections import ( - close_clone_targets, - connect_clone_targets, -) -from sqlbuild.cli.commands._helpers.clone.execution import execute_clone_plan -from sqlbuild.cli.commands._helpers.clone.invocation import resolve_clone_invocation -from sqlbuild.cli.commands._helpers.clone.outputs import ( - resolve_clone_exit_code, - write_clone_completion_output, - write_clone_execution_header, - write_clone_execution_json_output, -) -from sqlbuild.cli.commands._helpers.clone.planning import prepare_clone_execution -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneConnectionContext, - CloneExecutionPreparation, - CloneInvocation, - CloneRunOutcome, -) - - -def run_clone(request: CloneCommandRequest) -> int: - """Execute the clone command.""" - - invocation: CloneInvocation = resolve_clone_invocation(request=request) - connection_context: CloneConnectionContext = connect_clone_targets( - request=request, - invocation=invocation, - ) - try: - preparation: CloneExecutionPreparation = prepare_clone_execution( - request=request, - invocation=invocation, - connection_context=connection_context, - ) - write_clone_execution_header( - request=request, - invocation=invocation, - preparation=preparation, - ) - outcome: CloneRunOutcome = execute_clone_plan( - request=request, - invocation=invocation, - connection_context=connection_context, - preparation=preparation, - ) - finally: - _ = close_clone_targets(invocation=invocation, connection_context=connection_context) - write_clone_completion_output(invocation=invocation, outcome=outcome) - write_clone_execution_json_output( - request=request, - preparation=preparation, - outcome=outcome, - ) - return resolve_clone_exit_code(outcome) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 654d4ccd72..20537ecb7c 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -60,7 +60,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ( - ClonePipelineResult, CompilePipelineResult, PythonPlanEntry, ) @@ -68,7 +67,6 @@ from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan from sqlbuild.compiler.refactoring.models import RefactorProject from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult - from sqlbuild.executor.clone.models import CloneExecutionResult from sqlbuild.executor.diff.models import DiffExecutionResult, FullDiffSizeLimits from sqlbuild.executor.janitor.models import JanitorPlan from sqlbuild.executor.load.models import LoadExecutionResult @@ -337,42 +335,6 @@ class CloneCommandRequest: event_output_path: Path | None = None -@dataclass(frozen=True) -class CloneInvocation: - """Resolved project, adapter, and output context for clone.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter_name: str - adapter: BaseAdapter - destination_target_name: str - use_color: bool - progress_stream: TextIO - - -@dataclass(frozen=True) -class CloneConnectionContext: - """Destination connection configuration and handle for a clone.""" - - destination_connection_config: dict[str, object] - destination_connection: Any - - -@dataclass(frozen=True) -class CloneExecutionPreparation: - """Prepared direct clone pipeline and selected destination entries.""" - - pipeline_result: ClonePipelineResult - - -@dataclass(frozen=True) -class CloneRunOutcome: - """Direct clone execution result and elapsed time.""" - - result: CloneExecutionResult - elapsed: float - - @dataclass(frozen=True) class DbtSqlbuildWorkContext: """Shared execution context for one dbt interop SQLBuild work run.""" diff --git a/src/sqlbuild/compiler/pipeline/main/clone.py b/src/sqlbuild/compiler/pipeline/main/_clone.py similarity index 100% rename from src/sqlbuild/compiler/pipeline/main/clone.py rename to src/sqlbuild/compiler/pipeline/main/_clone.py diff --git a/src/sqlbuild/presentation/_helpers/structure.py b/src/sqlbuild/presentation/_helpers/structure.py index 89f34f3ef6..dfe6e82998 100644 --- a/src/sqlbuild/presentation/_helpers/structure.py +++ b/src/sqlbuild/presentation/_helpers/structure.py @@ -4,8 +4,6 @@ from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.constants import ( - PHASE_FAIL_GLYPH, - PHASE_OK_GLYPH, TREE_BRANCH_GLYPH, TREE_LAST_GLYPH, ) @@ -24,14 +22,3 @@ def format_surface_header(*, style: CliStyle, title: str, context: str | None = if context: rendered = f"{rendered} {style.muted(context)}" return rendered - - -def format_phase_line(*, style: CliStyle, ok: bool, label: str, summary: str | None = None) -> str: - """Render a concise phase-completion line: state glyph, label, dim summary.""" - - glyph: str = style.success(PHASE_OK_GLYPH) if ok else style.error(PHASE_FAIL_GLYPH) - rendered_label: str = label if ok else style.error_strong(label) - rendered: str = f"{glyph} {rendered_label}" - if summary: - rendered = f"{rendered} {style.muted(summary)}" - return rendered diff --git a/src/sqlbuild/presentation/main/phase_line.py b/src/sqlbuild/presentation/main/phase_line.py deleted file mode 100644 index 2c6b3796bc..0000000000 --- a/src/sqlbuild/presentation/main/phase_line.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public phase-completion line rendering entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.structure import format_phase_line as _format_phase_line -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_phase_line(*, style: CliStyle, ok: bool, label: str, summary: str | None = None) -> str: - """Render a concise phase-completion line: state glyph, label, dim summary.""" - - return _format_phase_line(style=style, ok=ok, label=label, summary=summary) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/__init__.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/__init__.py deleted file mode 100644 index 73b7c8bf6f..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Clone helper output tests.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py deleted file mode 100644 index 6457e8f576..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py +++ /dev/null @@ -1,57 +0,0 @@ -from dataclasses import dataclass - -from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult - - -@dataclass(frozen=True) -class RenderCloneOutputTestCase: - description: str - result: CloneExecutionResult - expected_fragments: tuple[str, ...] - expected_color_fragments: tuple[str, ...] = () - unexpected_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class RenderCloneItemLineTestCase: - description: str - index: int - total: int - item: CloneItemResult - expected_fragments: tuple[str, ...] - expected_line: str - relation_width: int | None = None - unexpected_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class CloneConnectionLifecycleTestCase: - description: str - expected_destination_config: dict[str, object] - - -@dataclass(frozen=True) -class RenderCloneFingerprintProgressTestCase: - description: str - completed: int - total: int - expected_line: str - - -@dataclass(frozen=True) -class RenderCloneFingerprintInterruptedTestCase: - description: str - completed: int - total: int - pending_identities: tuple[str, ...] - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class CloneFingerprintProgressReporterTestCase: - description: str - completed: int - total: int - pending_identities: tuple[str, ...] - expected_fragments: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_connections.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_connections.py deleted file mode 100644 index ed95052a97..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_connections.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import annotations - -from io import StringIO -from pathlib import Path -from typing import Any, cast - -import pytest - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.clone.connections import ( - close_clone_targets, - connect_clone_targets, -) -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneConnectionContext, - CloneInvocation, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig -from tests.unit.src.sqlbuild.cli.commands._helpers.clone._test_types import ( - CloneConnectionLifecycleTestCase, -) - - -class _ConnectionTrackingAdapter: - def __init__(self) -> None: - self.connected_configs: list[dict[str, object]] = [] - self.closed_connections: list[object] = [] - self.connection: object = object() - - def connect(self, config: dict[str, Any]) -> object: - self.connected_configs.append(config) - return self.connection - - def close(self, connection: object) -> None: - self.closed_connections.append(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - CloneConnectionLifecycleTestCase( - description="connects and closes only the destination session", - expected_destination_config={"database": "destination.duckdb"}, - ) - ], - ids=lambda case: case.description, -) -def test_given_clone_targets_when_connecting_then_only_destination_session_is_opened_once( - test_case: CloneConnectionLifecycleTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - adapter: _ConnectionTrackingAdapter = _ConnectionTrackingAdapter() - monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.clone.connections.resolve_target_connection_config", - lambda **_: test_case.expected_destination_config, - ) - invocation: CloneInvocation = CloneInvocation( - effective_project_dir=Path("."), - discovered_inputs=DiscoveredProjectInputs( - project_config=ProjectConfig(name="test", adapter="duckdb"), - local_config=LocalConfig(), - ), - adapter_name="duckdb", - adapter=cast(BaseAdapter, adapter), - destination_target_name="dev", - use_color=False, - progress_stream=StringIO(), - ) - request: CloneCommandRequest = CloneCommandRequest( - project_dir=None, - no_color=True, - no_sql_validation=False, - origin_target_name="prod", - destination_target_name="dev", - hard_copy=False, - ) - - context: CloneConnectionContext = connect_clone_targets(request=request, invocation=invocation) - close_clone_targets(invocation=invocation, connection_context=context) - - assert adapter.connected_configs == [test_case.expected_destination_config] - assert adapter.closed_connections == [adapter.connection] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_fingerprint_progress.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_fingerprint_progress.py deleted file mode 100644 index fb959f4d0e..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_fingerprint_progress.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -from io import StringIO - -import pytest - -from sqlbuild.cli.commands._helpers.clone.fingerprint_progress import ( - create_clone_fingerprint_progress_reporter, -) -from sqlbuild.executor.clone.types import CloneFingerprintProgressReporter -from tests.unit.src.sqlbuild.cli.commands._helpers.clone._test_types import ( - CloneFingerprintProgressReporterTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - CloneFingerprintProgressReporterTestCase( - description="reports completed batch and unconfirmed identities after interruption", - completed=50, - total=52, - pending_identities=("model:orders_50", "model:orders_51"), - expected_fragments=( - "[50/52] propagated OK", - "[50/52] FAIL not confirmed: model:orders_50, model:orders_51", - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_partial_progress_when_interrupted_then_reporter_names_unconfirmed_fingerprints( - test_case: CloneFingerprintProgressReporterTestCase, -) -> None: - stream: StringIO = StringIO() - reporter: CloneFingerprintProgressReporter = create_clone_fingerprint_progress_reporter( - stream=stream, - use_color=False, - ) - - reporter( - completed=0, - total=test_case.total, - pending_identities=("model:orders_0", *test_case.pending_identities), - ) - reporter( - completed=test_case.completed, - total=test_case.total, - pending_identities=test_case.pending_identities, - ) - reporter.write_interrupted() - - output: str = stream.getvalue() - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_output.py deleted file mode 100644 index 896b49a3ac..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_output.py +++ /dev/null @@ -1,228 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.clone.output import ( - render_clone_fingerprint_interrupted_line, - render_clone_fingerprint_progress_line, - render_clone_item_line, - render_clone_output, -) -from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult -from sqlbuild.executor.clone.types import CloneAction, CloneStatus -from tests.unit.src.sqlbuild.cli.commands._helpers.clone._test_types import ( - RenderCloneFingerprintInterruptedTestCase, - RenderCloneFingerprintProgressTestCase, - RenderCloneItemLineTestCase, - RenderCloneOutputTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - RenderCloneFingerprintProgressTestCase( - description="confirmed batch reports cumulative progress", - completed=50, - total=386, - expected_line=" fingerprint[ 50/386] propagated OK", - ), - ), - ids=lambda case: case.description, -) -def test_given_confirmed_fingerprint_batch_when_rendering_then_progress_is_explicit( - test_case: RenderCloneFingerprintProgressTestCase, -) -> None: - rendered: str = render_clone_fingerprint_progress_line( - completed=test_case.completed, - total=test_case.total, - use_color=False, - ) - - assert rendered == test_case.expected_line - - -@pytest.mark.parametrize( - "test_case", - ( - RenderCloneFingerprintInterruptedTestCase( - description="interruption bounds pending identity context", - completed=50, - total=62, - pending_identities=tuple(f"model:orders_{index}" for index in range(12)), - expected_fragments=("[50/62] FAIL", "model:orders_0", "model:orders_9", "(+2 more)"), - unexpected_fragments=("orders_10",), - ), - ), - ids=lambda case: case.description, -) -def test_given_interrupted_fingerprint_batch_when_rendering_then_pending_context_is_bounded( - test_case: RenderCloneFingerprintInterruptedTestCase, -) -> None: - rendered: str = render_clone_fingerprint_interrupted_line( - completed=test_case.completed, - total=test_case.total, - pending_identities=test_case.pending_identities, - use_color=False, - ) - - assert all(fragment in rendered for fragment in test_case.expected_fragments) - assert all(fragment not in rendered for fragment in test_case.unexpected_fragments) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderCloneOutputTestCase( - description="summary footer reports counts and surfaces non-success messages", - result=CloneExecutionResult( - item_results=( - CloneItemResult( - name="fact_orders", - action=CloneAction.CLONED, - status=CloneStatus.SUCCESS, - ), - CloneItemResult( - name="stg_orders", - action=CloneAction.WARNING_MISSING_SOURCE, - status=CloneStatus.WARNING, - message="missing in origin environment", - ), - CloneItemResult( - name="dim_customers", - action=CloneAction.FAILED, - status=CloneStatus.FAILED, - message="boom", - ), - CloneItemResult( - name="add_tax", - action=CloneAction.RECREATED_FUNCTION, - status=CloneStatus.SUCCESS, - ), - ) - ), - expected_fragments=( - "stg_orders", - "missing in origin environment", - "dim_customers", - "boom", - "RECREATED_FUNCTIONS=1", - "\u2717 Completed with errors", - ), - expected_color_fragments=( - "\033[38;5;167m\u2717\033[0m \033[38;5;167m\033[1mCompleted with errors\033[0m", - "\033[2mCLONED=\033[0m\033[34m1\033[0m", - "\033[2mPASS=\033[0m\033[32m2\033[0m", - "\033[2mWARN=\033[0m\033[33m1\033[0m", - "\033[2mFAIL=\033[0m\033[38;5;167m1\033[0m", - ), - unexpected_fragments=("fact_orders", "add_tax"), - ), - ], - ids=lambda case: case.description, -) -def test_given_clone_result_when_rendering_summary_then_reports_footer_and_messages( - test_case: RenderCloneOutputTestCase, - capsys: pytest.CaptureFixture[str], -) -> None: - render_clone_output(result=test_case.result, use_color=False) - no_color_output: str = capsys.readouterr().out - - render_clone_output(result=test_case.result, use_color=True) - color_output: str = capsys.readouterr().out - - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in no_color_output - for fragment in test_case.unexpected_fragments: - assert fragment not in no_color_output - assert "\033[" not in no_color_output - for fragment in test_case.expected_color_fragments: - assert fragment in color_output - - -@pytest.mark.parametrize( - "test_case", - [ - RenderCloneItemLineTestCase( - description="streamed line shows position, action, origin->destination and status", - index=2, - total=5, - item=CloneItemResult( - name="fact_orders", - action=CloneAction.COPIED, - status=CloneStatus.SUCCESS, - origin_relation="PROD.MAIN.FACT_ORDERS", - destination_relation="DEV.MAIN.FACT_ORDERS", - duration_seconds=0.42, - ), - expected_fragments=( - "2/5", - "copied", - "prod.main.fact_orders", - "->", - "dev.main.fact_orders", - "OK", - "0.42s", - ), - relation_width=50, - expected_line=( - " 2/5 copied " - "prod.main.fact_orders -> dev.main.fact_orders OK 0.42s" - ), - unexpected_fragments=("PROD.MAIN.FACT_ORDERS", "DEV.MAIN.FACT_ORDERS"), - ), - RenderCloneItemLineTestCase( - description="mixed quoted relation names preserve quoted case", - index=5, - total=5, - item=CloneItemResult( - name="orders", - action=CloneAction.CLONED, - status=CloneStatus.SUCCESS, - origin_relation='PROD.MAIN."Orders"', - destination_relation='DEV.MAIN."Orders"', - ), - expected_fragments=('prod.main."Orders"', 'dev.main."Orders"'), - expected_line=( - ' 5/5 cloned prod.main."Orders" -> dev.main."Orders" OK' - ), - unexpected_fragments=("PROD.MAIN", "DEV.MAIN"), - ), - RenderCloneItemLineTestCase( - description="escaped bracket identifier preserves quoted case", - index=5, - total=5, - item=CloneItemResult( - name="orders", - action=CloneAction.CLONED, - status=CloneStatus.SUCCESS, - origin_relation="PROD.DBO.[My]]Table]", - destination_relation="DEV.DBO.[My]]Table]", - ), - expected_fragments=("prod.dbo.[My]]Table]", "dev.dbo.[My]]Table]"), - expected_line=( - " 5/5 cloned prod.dbo.[My]]Table] -> dev.dbo.[My]]Table] OK" - ), - unexpected_fragments=("[My]]table]",), - ), - ], - ids=lambda case: case.description, -) -def test_given_clone_item_when_rendering_line_then_shows_flow_and_status( - test_case: RenderCloneItemLineTestCase, -) -> None: - line: str = render_clone_item_line( - index=test_case.index, - total=test_case.total, - item=test_case.item, - use_color=False, - relation_width=test_case.relation_width, - ) - - assert line == test_case.expected_line - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in line - for fragment in test_case.unexpected_fragments: - assert fragment not in line From ac2b574aaf5f652b1b9480205e39d5be1c37447b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:21:40 +0100 Subject: [PATCH 02/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 253fe17c20..d647cf1141 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 89820 +remaining_lines = 88829 [[stays]] path = "executor" From 73124f7f84338fe288a5d3cdc7b698bb513e713a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:12:06 +0100 Subject: [PATCH 03/37] refactor(state): build migration event IDs, event SQL and freshness versions natively The new plain-Rust sqlbuild-state crate computes the content-addressed migration, column-rename and old-name view event IDs, renders the idempotent event lookup and typed INSERT statements, and normalizes, hashes, compares and age-checks source freshness data versions. Stored IDs and hashes are pinned to values the Python implementation wrote. Reads and writes still go through the adapter, and warehouse orchestration stays in Python. --- Cargo.lock | 10 + Cargo.toml | 3 + crates/README.md | 3 +- crates/sqlbuild-python/Cargo.toml | 1 + .../_helpers/freshness/freshness_observe.rs | 35 ++- .../src/bindings/_helpers/mod.rs | 1 + .../bindings/_helpers/state_rows/events.rs | 164 ++++++++++++ .../bindings/_helpers/state_rows/freshness.rs | 237 ++++++++++++++++++ .../src/bindings/_helpers/state_rows/mod.rs | 4 + .../_helpers/state_rows/registration.rs | 11 + .../_helpers/state_rows/timestamps.rs | 33 +++ .../src/bindings/main/register.rs | 3 +- crates/sqlbuild-state/Cargo.toml | 16 ++ .../src/events/_helpers/identity.rs | 49 ++++ .../src/events/_helpers/literals.rs | 46 ++++ .../sqlbuild-state/src/events/_helpers/mod.rs | 2 + crates/sqlbuild-state/src/events/constants.rs | 12 + crates/sqlbuild-state/src/events/errors.rs | 15 ++ .../events/main/column_migration_event_id.rs | 25 ++ .../src/events/main/event_exists_sql.rs | 12 + .../src/events/main/event_insert_sql.rs | 19 ++ .../src/events/main/migration_event_id.rs | 22 ++ crates/sqlbuild-state/src/events/main/mod.rs | 5 + .../src/events/main/old_name_view_event_id.rs | 20 ++ crates/sqlbuild-state/src/events/mod.rs | 8 + crates/sqlbuild-state/src/events/models.rs | 37 +++ crates/sqlbuild-state/src/events/tests.rs | 6 + .../src/events/tests/helpers.rs | 9 + .../src/events/tests/test_insert_sql.rs | 47 ++++ .../src/events/tests/test_types.rs | 8 + .../src/freshness/_helpers/durations.rs | 34 +++ .../src/freshness/_helpers/mod.rs | 2 + .../src/freshness/_helpers/moments.rs | 42 ++++ .../sqlbuild-state/src/freshness/constants.rs | 30 +++ crates/sqlbuild-state/src/freshness/errors.rs | 24 ++ .../src/freshness/main/age_status.rs | 41 +++ .../src/freshness/main/data_version_hash.rs | 37 +++ .../sqlbuild-state/src/freshness/main/mod.rs | 4 + .../freshness/main/normalized_data_version.rs | 25 ++ .../src/freshness/main/records_equivalent.rs | 63 +++++ crates/sqlbuild-state/src/freshness/mod.rs | 9 + crates/sqlbuild-state/src/freshness/models.rs | 32 +++ crates/sqlbuild-state/src/freshness/tests.rs | 6 + .../src/freshness/tests/helpers.rs | 31 +++ .../src/freshness/tests/test_equivalence.rs | 56 +++++ .../src/freshness/tests/test_types.rs | 12 + crates/sqlbuild-state/src/freshness/types.rs | 11 + crates/sqlbuild-state/src/lib.rs | 6 + fensu.toml | 1 + src/sqlbuild/_native.pyi | 62 ++++- .../migrations/_helpers/column_sql.py | 3 +- .../migrations/_helpers/old_name_sql.py | 3 +- .../compiler/migrations/_helpers/sql.py | 30 +-- .../main/deterministic_column_event_id.py | 30 --- .../migrations/main/deterministic_event_id.py | 27 -- .../deterministic_old_name_view_event_id.py | 18 -- .../source_freshness/_helpers/age_policy.py | 47 ---- .../source_freshness/_helpers/planning.py | 18 +- .../source_freshness/_helpers/state.py | 112 --------- .../source_freshness/main/age_policy.py | 26 -- .../main/data_version_hash.py | 25 -- .../source_freshness/main/normalization.py | 16 -- .../main/record_equivalence.py | 23 -- .../executor/migrations/_helpers/columns.py | 4 +- .../migrations/_helpers/old_name_facts.py | 4 +- .../migrations/_helpers/old_name_views.py | 4 +- .../executor/migrations/_helpers/promotion.py | 4 +- .../old_name_views/_helpers/janitor.py | 4 +- .../sqlbuild/adapters/bigquery/test_client.py | 13 +- .../adapters/databricks/test_client.py | 13 +- .../adapters/snowflake/test_client.py | 10 +- .../commands/_helpers/freshness/helpers.py | 4 +- .../compiler/migrations/main/_test_types.py | 40 +++ .../compiler/migrations/main/helpers.py | 6 +- .../migrations/main/test_pinned_event_ids.py | 137 ++++++++++ .../_helpers/test_age_policy.py | 4 +- .../test_observation.py | 4 +- .../compiler/source_freshness/main/helpers.py | 4 +- .../source_freshness/main/test_age_policy.py | 4 +- .../source_freshness/main/test_planning.py | 4 +- .../source_freshness/main/test_state.py | 8 +- .../executor/migrations/_helpers/helpers.py | 4 +- 82 files changed, 1588 insertions(+), 456 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs create mode 100644 crates/sqlbuild-state/Cargo.toml create mode 100644 crates/sqlbuild-state/src/events/_helpers/identity.rs create mode 100644 crates/sqlbuild-state/src/events/_helpers/literals.rs create mode 100644 crates/sqlbuild-state/src/events/_helpers/mod.rs create mode 100644 crates/sqlbuild-state/src/events/constants.rs create mode 100644 crates/sqlbuild-state/src/events/errors.rs create mode 100644 crates/sqlbuild-state/src/events/main/column_migration_event_id.rs create mode 100644 crates/sqlbuild-state/src/events/main/event_exists_sql.rs create mode 100644 crates/sqlbuild-state/src/events/main/event_insert_sql.rs create mode 100644 crates/sqlbuild-state/src/events/main/migration_event_id.rs create mode 100644 crates/sqlbuild-state/src/events/main/mod.rs create mode 100644 crates/sqlbuild-state/src/events/main/old_name_view_event_id.rs create mode 100644 crates/sqlbuild-state/src/events/mod.rs create mode 100644 crates/sqlbuild-state/src/events/models.rs create mode 100644 crates/sqlbuild-state/src/events/tests.rs create mode 100644 crates/sqlbuild-state/src/events/tests/helpers.rs create mode 100644 crates/sqlbuild-state/src/events/tests/test_insert_sql.rs create mode 100644 crates/sqlbuild-state/src/events/tests/test_types.rs create mode 100644 crates/sqlbuild-state/src/freshness/_helpers/durations.rs create mode 100644 crates/sqlbuild-state/src/freshness/_helpers/mod.rs create mode 100644 crates/sqlbuild-state/src/freshness/_helpers/moments.rs create mode 100644 crates/sqlbuild-state/src/freshness/constants.rs create mode 100644 crates/sqlbuild-state/src/freshness/errors.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/age_status.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/data_version_hash.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/mod.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/normalized_data_version.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/records_equivalent.rs create mode 100644 crates/sqlbuild-state/src/freshness/mod.rs create mode 100644 crates/sqlbuild-state/src/freshness/models.rs create mode 100644 crates/sqlbuild-state/src/freshness/tests.rs create mode 100644 crates/sqlbuild-state/src/freshness/tests/helpers.rs create mode 100644 crates/sqlbuild-state/src/freshness/tests/test_equivalence.rs create mode 100644 crates/sqlbuild-state/src/freshness/tests/test_types.rs create mode 100644 crates/sqlbuild-state/src/freshness/types.rs create mode 100644 crates/sqlbuild-state/src/lib.rs delete mode 100644 src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py delete mode 100644 src/sqlbuild/compiler/migrations/main/deterministic_event_id.py delete mode 100644 src/sqlbuild/compiler/migrations/main/deterministic_old_name_view_event_id.py delete mode 100644 src/sqlbuild/compiler/source_freshness/_helpers/age_policy.py delete mode 100644 src/sqlbuild/compiler/source_freshness/_helpers/state.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/age_policy.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/data_version_hash.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/normalization.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/record_equivalence.py create mode 100644 tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py diff --git a/Cargo.lock b/Cargo.lock index af826cf0c7..7630404f05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1274,6 +1274,7 @@ dependencies = [ "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", + "sqlbuild-state", ] [[package]] @@ -1344,6 +1345,15 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "sqlbuild-state" +version = "0.156.1" +dependencies = [ + "sha2", + "sqlbuild-core", + "sqlbuild-cursor", +] + [[package]] name = "sqlparser" version = "0.58.0" diff --git a/Cargo.toml b/Cargo.toml index ad79fd0a8e..1eed898f30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/sqlbuild-core", "crates/sqlbuild-cursor", + "crates/sqlbuild-state", "crates/sqlbuild-pysource", "crates/sqlbuild-cache", "crates/sqlbuild-sqltext", @@ -26,6 +27,7 @@ resolver = "2" order = [ "sqlbuild-core", "sqlbuild-cursor", + "sqlbuild-state", "sqlbuild-pysource", "sqlbuild-cache", "sqlbuild-sqltext", @@ -104,6 +106,7 @@ sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } +sqlbuild-state = { path = "crates/sqlbuild-state" } stacker = "0.1" sqlparser = { version = "0.58", features = ["visitor"] } syn = { version = "2", features = ["full", "visit", "extra-traits"] } diff --git a/crates/README.md b/crates/README.md index f957e37ee2..b1f38540f6 100644 --- a/crates/README.md +++ b/crates/README.md @@ -8,6 +8,7 @@ that can be tested and benchmarked with `cargo test` alone. |---|---| | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | | `sqlbuild-cursor` | Microbatch cursor algebra: typed cursor values, grain flooring and stepping, half-open interval clamping, merging and batch splitting, timestamp text normalization and calendar durations, with Python `datetime` results and errors. Zones other than fixed offsets are resolved through a caller-supplied lookup. | +| `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence and age policy, byte-compatible with the rows Python wrote. | | `sqlbuild-pysource` | Static reading of project Python source with the ruff parser: macro module structure, literal SQL arguments, definition locations and string-literal edits. It never imports or runs user code; CPython stays the authority on whether a file is valid Python. | | `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. | @@ -31,7 +32,7 @@ Dependencies point one way, from the top of this graph to the bottom: sqlbuild-python -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-config - -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-cursor -> sqlbuild-core + -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core sqlbuild-cli -> sqlbuild-cache, sqlbuild-core ``` diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index fc2f00f2b3..0ac50aadbd 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -36,6 +36,7 @@ sqlbuild-render.workspace = true sqlbuild-rules.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true +sqlbuild-state.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs index 79536ecd32..2657a8e5bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs @@ -17,6 +17,8 @@ use sqlbuild_core::freshness_text::models::{ }; const FRESHNESS_PACKAGE: &str = "sqlbuild.compiler.source_freshness.main"; +const NATIVE: &str = "_native"; +const NATIVE_MODULE: &str = "sqlbuild._native"; const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; @@ -44,6 +46,9 @@ fn freshness_function<'py>( py: Python<'py>, (module, name): (&str, &str), ) -> PyResult> { + if module == NATIVE { + return py.import(NATIVE_MODULE)?.getattr(name); + } py.import(format!("{FRESHNESS_PACKAGE}.{module}"))? .getattr(name) } @@ -131,21 +136,17 @@ fn record<'py>( let normalize: Bound<'py, PyDict> = PyDict::new(py); normalize.set_item("value", observation.getattr("data_version")?)?; normalize.set_item("value_kind", observation.getattr("value_kind")?)?; - let data_version: Bound<'py, PyAny> = freshness_function( - py, - ("normalization", "normalize_source_freshness_data_version"), - )? - .call((), Some(&normalize))?; + let data_version: Bound<'py, PyAny> = + freshness_function(py, (NATIVE, "normalize_source_freshness_data_version"))? + .call((), Some(&normalize))?; let hash: Bound<'py, PyDict> = PyDict::new(py); hash.set_item("source_name", observation.getattr("source_name")?)?; hash.set_item("strategy", observation.getattr("strategy")?)?; hash.set_item("value_kind", observation.getattr("value_kind")?)?; hash.set_item("data_version", &data_version)?; - let data_version_hash: Bound<'py, PyAny> = freshness_function( - py, - ("data_version_hash", "source_freshness_data_version_hash"), - )? - .call((), Some(&hash))?; + let data_version_hash: Bound<'py, PyAny> = + freshness_function(py, (NATIVE, "source_freshness_data_version_hash"))? + .call((), Some(&hash))?; let fields: Bound<'py, PyDict> = PyDict::new(py); fields.set_item("source_name", observation.getattr("source_name")?)?; fields.set_item("target_database", source.getattr("database")?)?; @@ -161,7 +162,7 @@ fn record<'py>( observation.getattr("value_kind")?.getattr("value")?, )?; fields.set_item("data_version", &data_version)?; - fields.set_item("data_version_hash", data_version_hash)?; + fields.set_item(NATIVE, data_version_hash)?; fields.set_item("observed_at", observation.getattr("observed_at")?)?; py.import(FRESHNESS_MODELS)? .getattr("SourceFreshnessRecord")? @@ -200,7 +201,7 @@ fn observed_result( age.set_item("data_version", observation.getattr("data_version")?)?; age.set_item("observed_at", observation.getattr("observed_at")?)?; let age_status: Bound<'_, PyAny> = - freshness_function(py, ("age_policy", "evaluate_source_freshness_age_policy"))? + freshness_function(py, (NATIVE, "evaluate_source_freshness_age_policy"))? .call((), Some(&age))?; let previous: Option> = previous_record(state, ¤t)?; let comparison: StateComparison = match &previous { @@ -211,12 +212,10 @@ fn observed_result( options.set_item("previous_record", previous)?; options.set_item("current_record", ¤t)?; options.set_item("lag_tolerance", &lag_tolerance)?; - let equivalent: bool = freshness_function( - py, - ("record_equivalence", "source_freshness_records_equivalent"), - )? - .call((), Some(&options))? - .is_truthy()?; + let equivalent: bool = + freshness_function(py, (NATIVE, "source_freshness_records_equivalent"))? + .call((), Some(&options))? + .is_truthy()?; if equivalent { StateComparison::Equivalent { same_hash: previous diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index ecd35332b7..8ef9476d7f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -43,5 +43,6 @@ pub(crate) mod semantic_checks; pub(crate) mod semantic_shapes; pub(crate) mod sql_test_glue; pub(crate) mod sqltext; +pub(crate) mod state_rows; pub(crate) mod type_system; pub(crate) mod workspace_commands; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs new file mode 100644 index 0000000000..5b0c5d4001 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs @@ -0,0 +1,164 @@ +//! Migration event IDs and the SQL that appends and looks up one event row. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_state::events::errors::StateSqlError; +use sqlbuild_state::events::main::column_migration_event_id::column_migration_event_id; +use sqlbuild_state::events::main::event_exists_sql::event_exists_sql; +use sqlbuild_state::events::main::event_insert_sql::event_insert_sql; +use sqlbuild_state::events::main::migration_event_id::migration_event_id; +use sqlbuild_state::events::main::old_name_view_event_id::old_name_view_event_id; +use sqlbuild_state::events::models::{EventRelation, StateCell, StateColumnType, StateValue}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, presumed_utc}; + +const SQL_VALUES_EXCEPTIONS: &str = "sqlbuild.sql_values.exceptions"; +const STRING_TYPE: &str = "string"; +const TEXT_TIMESTAMP_TYPE: &str = "text_timestamp"; + +fn emit_error(error: JsonEmitError) -> PyErr { + PyValueError::new_err(format!("{error:?}")) +} + +fn relation(value: &Bound<'_, PyAny>) -> PyResult { + Ok(EventRelation { + database: value.getattr("database")?.extract()?, + schema: value.getattr("schema")?.extract()?, + name: value.getattr("name")?.extract()?, + }) +} + +/// The content-addressed ID for one migration performed by one run. +#[pyfunction(name = "deterministic_migration_event_id")] +#[pyo3(signature = (*, run_id, target_name, origin, destination))] +fn migration_id( + run_id: &str, + target_name: Option<&str>, + origin: &Bound<'_, PyAny>, + destination: &Bound<'_, PyAny>, +) -> PyResult { + migration_event_id( + (run_id, target_name), + &relation(origin)?, + &relation(destination)?, + ) + .map_err(emit_error) +} + +/// The content-addressed ID for one column rename performed by one run. +#[pyfunction(name = "deterministic_column_migration_event_id")] +#[pyo3(signature = (**request))] +fn column_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("deterministic_column_migration_event_id", request)?; + let run_id: String = request.get("run_id")?.extract()?; + let target_name: Option = request.get("target_name")?.extract()?; + let origin_column: String = request.get("origin_column")?.extract()?; + let destination_column: String = request.get("destination_column")?.extract()?; + column_migration_event_id( + (&run_id, target_name.as_deref()), + &relation(&request.get("relation")?)?, + (&origin_column, &destination_column), + ) + .map_err(emit_error) +} + +/// The content-addressed ID of one old-name step of one recorded move. +#[pyfunction(name = "deterministic_old_name_view_event_id")] +#[pyo3(signature = (*, event_type, migration_event_id))] +fn old_name_id(event_type: &Bound<'_, PyAny>, migration_event_id: &str) -> PyResult { + let event_type: String = event_type.getattr("value")?.extract()?; + old_name_view_event_id(&event_type, migration_event_id).map_err(emit_error) +} + +/// The lookup that makes an append-only event write idempotent. +#[pyfunction(name = "render_event_exists_sql")] +#[pyo3(signature = (*, table, event_id))] +fn exists_sql(table: &str, event_id: &str) -> String { + event_exists_sql(table, event_id) +} + +fn state_value(value: &Bound<'_, PyAny>, column_type: StateColumnType) -> PyResult { + if value.is_none() { + return Ok(StateValue::Null); + } + if let Ok(text) = value.downcast::() { + return Ok(StateValue::Text(text.extract()?)); + } + if column_type == StateColumnType::TextTimestamp && date_time_typed(value)? { + return Ok(StateValue::UtcTimestamp(presumed_utc(value)?)); + } + Ok(StateValue::Other) +} + +fn column_type(declared: &Bound<'_, PyAny>) -> PyResult { + let name: String = declared.str()?.extract()?; + match name.as_str() { + STRING_TYPE => Ok(StateColumnType::String), + TEXT_TIMESTAMP_TYPE => Ok(StateColumnType::TextTimestamp), + _ => Err(PyValueError::new_err(format!( + "migration event columns are string or text timestamps, not {name}" + ))), + } +} + +/// One append-only event row as `INSERT ... VALUES` with typed literals. +#[pyfunction(name = "render_event_insert_sql")] +#[pyo3(signature = (*, table, columns, column_types, values))] +fn insert_sql( + table: &str, + columns: Vec, + column_types: &Bound<'_, PyAny>, + values: Vec>, +) -> PyResult { + if columns.len() != values.len() { + return Err(PyValueError::new_err( + "event columns and values must have the same length", + )); + } + let mut cells: Vec = Vec::with_capacity(columns.len()); + for (column, value) in columns.into_iter().zip(&values) { + let column_type: StateColumnType = column_type(&column_types.get_item(&column)?)?; + let converted: PyResult = state_value(value, column_type); + let value: StateValue = match converted { + Ok(value) => value, + Err(error) => { + event_insert_sql(table, &cells).map_err(rendering_error(value.py()))?; + return Err(error); + } + }; + cells.push(StateCell { + column, + column_type, + value, + }); + } + event_insert_sql(table, &cells).map_err(rendering_error(column_types.py())) +} + +/// `SqlValueRenderingError(message)` for an ill-typed value. +fn rendering_error(py: Python<'_>) -> impl Fn(StateSqlError) -> PyErr + '_ { + move |error: StateSqlError| { + let raised = imported(py, SQL_VALUES_EXCEPTIONS, "SqlValueRenderingError") + .and_then(|class| class.call1((error.0,))); + match raised { + Ok(exception) => PyErr::from_value(exception), + Err(failure) => failure, + } + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(migration_id, module)?)?; + module.add_function(wrap_pyfunction!(column_id, module)?)?; + module.add_function(wrap_pyfunction!(old_name_id, module)?)?; + module.add_function(wrap_pyfunction!(exists_sql, module)?)?; + module.add_function(wrap_pyfunction!(insert_sql, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs new file mode 100644 index 0000000000..a2377ff690 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs @@ -0,0 +1,237 @@ +//! Source freshness data versions, their hashes, skip equivalence and age policy. + +use std::cell::RefCell; + +use pyo3::exceptions::PyOverflowError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyInt, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_state::freshness::errors::FreshnessError; +use sqlbuild_state::freshness::main::age_status::age_status; +use sqlbuild_state::freshness::main::data_version_hash::data_version_hash; +use sqlbuild_state::freshness::main::normalized_data_version::normalized_data_version; +use sqlbuild_state::freshness::main::records_equivalent::records_equivalent; +use sqlbuild_state::freshness::models::{AgeStatus, ComparableRecord, DataVersion}; +use sqlbuild_state::freshness::types::TimestampParser; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::values::{DATETIME_MODULE, local_date_time}; +use crate::bindings::_helpers::state_rows::timestamps::{ + date_time_typed, in_utc, offset_aware, presumed_utc, +}; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.source_freshness.exceptions"; +const TYPES_MODULE: &str = "sqlbuild.compiler.source_freshness.types"; +const TIMESTAMP_KIND: &str = "timestamp"; +const INTEGER_KIND: &str = "integer"; +const STRING_KIND: &str = "string"; + +fn raised(py: Python<'_>, class: &str, message: String) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, class).and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn python_error(py: Python<'_>, error: FreshnessError) -> PyErr { + match error { + FreshnessError::Observation(message) => { + raised(py, "SourceFreshnessObservationError", message) + } + FreshnessError::Input(message) => raised(py, "SourceFreshnessInputError", message), + FreshnessError::Overflow(message) => PyOverflowError::new_err(message), + } +} + +/// Hash the stable source freshness identity used as a graph input. +#[pyfunction(name = "source_freshness_data_version_hash")] +#[pyo3(signature = (*, source_name, strategy, value_kind, data_version))] +fn version_hash( + source_name: &str, + strategy: &str, + value_kind: &str, + data_version: &str, +) -> PyResult { + data_version_hash((source_name, strategy), value_kind, data_version) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(format!("{error:?}"))) +} + +fn observation_error(py: Python<'_>, message: &str) -> PyErr { + raised(py, "SourceFreshnessObservationError", message.to_owned()) +} + +/// A source freshness value normalized for stable state storage. +#[pyfunction(name = "normalize_source_freshness_data_version")] +#[pyo3(signature = (*, value, value_kind))] +fn normalize<'py>( + value: &Bound<'py, PyAny>, + value_kind: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = value.py(); + if value_kind.eq(TIMESTAMP_KIND)? { + if !date_time_typed(value)? { + return Err(observation_error( + py, + "Timestamp source freshness values must be datetime values", + )); + } + let aware: bool = !value.getattr("tzinfo")?.is_none(); + let local: LocalDateTime = if aware { + local_date_time(&in_utc(value)?)? + } else { + local_date_time(value)? + }; + let text: String = normalized_data_version(&DataVersion::Timestamp { local, utc: aware }); + return Ok(PyString::new(py, &text).into_any()); + } + if value_kind.eq(INTEGER_KIND)? { + if value.is_instance_of::() || !value.is_instance_of::() { + return Err(observation_error( + py, + "Integer source freshness values must be integer values", + )); + } + return Ok(value.str()?.into_any()); + } + if value_kind.eq(STRING_KIND)? { + if !value.is_instance_of::() { + return Err(observation_error( + py, + "String source freshness values must be string values", + )); + } + return Ok(value.clone()); + } + Err(raised( + py, + "SourceFreshnessObservationError", + format!( + "Unsupported source freshness value kind: {}", + value_kind.str()? + ), + )) +} + +/// Stored timestamp text parsed with `datetime.fromisoformat`, remembering a Python failure. +struct StateTimestamps<'py> { + py: Python<'py>, + failure: RefCell>, +} + +impl StateTimestamps<'_> { + fn parsed(&self, text: &str) -> PyResult { + let parsed = + imported(self.py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (text,)); + match parsed { + Ok(value) => presumed_utc(&value), + Err(error) if error.is_instance_of::(self.py) => { + let wrapped: PyErr = observation_error( + self.py, + &format!( + "Timestamp source freshness state value is not valid ISO datetime: {text}" + ), + ); + wrapped.set_cause(self.py, Some(error)); + Err(wrapped) + } + Err(error) => Err(error), + } + } +} + +impl TimestampParser for StateTimestamps<'_> { + fn utc_timestamp(&self, text: &str) -> Result { + self.parsed(text).map_err(|error| { + let message: String = error.to_string(); + *self.failure.borrow_mut() = Some(error); + FreshnessError::Observation(message) + }) + } +} + +fn comparable(record: &Bound<'_, PyAny>) -> PyResult { + Ok(ComparableRecord { + data_version_hash: record.getattr("data_version_hash")?.extract()?, + value_kind: record.getattr("value_kind")?.extract()?, + data_version: record.getattr("data_version")?.extract()?, + }) +} + +/// Whether two source freshness records are equivalent for skip decisions. +#[pyfunction(name = "source_freshness_records_equivalent")] +#[pyo3(signature = (*, previous_record, current_record, lag_tolerance = None))] +fn equivalent( + previous_record: &Bound<'_, PyAny>, + current_record: &Bound<'_, PyAny>, + lag_tolerance: Option<&str>, +) -> PyResult { + let py: Python<'_> = previous_record.py(); + let parser = StateTimestamps { + py, + failure: RefCell::new(None), + }; + let records = (&comparable(previous_record)?, &comparable(current_record)?); + records_equivalent(records, lag_tolerance, &parser).map_err(|error| { + parser + .failure + .borrow_mut() + .take() + .unwrap_or_else(|| python_error(py, error)) + }) +} + +/// A `datetime` required to be aware, in UTC. +fn aware_utc(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult { + if !offset_aware(value)? { + return Err(observation_error( + value.py(), + &format!("source freshness {field_name} must be timezone-aware"), + )); + } + local_date_time(&in_utc(value)?) +} + +/// Optional timestamp age policy for one source freshness observation. +#[pyfunction(name = "evaluate_source_freshness_age_policy")] +#[pyo3(signature = (*, policy, data_version, observed_at))] +fn age_policy<'py>( + policy: &Bound<'py, PyAny>, + data_version: &Bound<'py, PyAny>, + observed_at: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = policy.py(); + if policy.is_none() { + return Ok(None); + } + let statuses: Bound<'py, PyAny> = imported(py, TYPES_MODULE, "SourceFreshnessAgeStatus")?; + if !date_time_typed(data_version)? { + return statuses.getattr("UNKNOWN").map(Some); + } + let version: LocalDateTime = aware_utc(data_version, "data_version")?; + let observed: LocalDateTime = aware_utc(observed_at, "observed_at")?; + let error_after: Option = policy.getattr("error_after")?.extract()?; + let warn_after: Option = policy.getattr("warn_after")?.extract()?; + let status: AgeStatus = age_status( + (&version, &observed), + error_after.as_deref(), + warn_after.as_deref(), + ) + .map_err(|error| python_error(py, error))?; + let member: &str = match status { + AgeStatus::Pass => "PASS", + AgeStatus::Warn => "WARN", + AgeStatus::Error => "ERROR", + }; + statuses.getattr(member).map(Some) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(version_hash, module)?)?; + module.add_function(wrap_pyfunction!(normalize, module)?)?; + module.add_function(wrap_pyfunction!(equivalent, module)?)?; + module.add_function(wrap_pyfunction!(age_policy, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs new file mode 100644 index 0000000000..936e0ba0fa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod events; +pub(crate) mod freshness; +pub(crate) mod registration; +pub(crate) mod timestamps; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs new file mode 100644 index 0000000000..c06548943e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs @@ -0,0 +1,11 @@ +//! Register the native warehouse state row functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::state_rows::{events, freshness}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + events::register(module)?; + freshness::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs new file mode 100644 index 0000000000..b679778b32 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs @@ -0,0 +1,33 @@ +//! Python `datetime` values normalized to UTC wall time. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::values::{DATETIME_MODULE, local_date_time}; + +/// Whether `value` is a `datetime`. +pub(crate) fn date_time_typed(value: &Bound<'_, PyAny>) -> PyResult { + value.is_instance(&imported(value.py(), DATETIME_MODULE, "datetime")?) +} + +/// Whether `value` has a `tzinfo` that reports an offset. +pub(crate) fn offset_aware(value: &Bound<'_, PyAny>) -> PyResult { + Ok(!value.getattr("tzinfo")?.is_none() && !value.call_method0("utcoffset")?.is_none()) +} + +/// `value.astimezone(UTC)`. +pub(crate) fn in_utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + value.call_method1( + "astimezone", + (imported(value.py(), DATETIME_MODULE, "UTC")?,), + ) +} + +/// The UTC wall time of `value`, reading a naive or offset-less value as UTC. +pub(crate) fn presumed_utc(value: &Bound<'_, PyAny>) -> PyResult { + if offset_aware(value)? { + return local_date_time(&in_utc(value)?); + } + local_date_time(value) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index e42c8d671b..37ec76b7b2 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::{ fingerprints, freshness, frontier, graph, janitor, lineage, macro_bridge, manifest, model_config, plan_output, presentation_text, project_assembly, refactor, references, render, result_output, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, - type_system, workspace_commands, + state_rows, type_system, workspace_commands, }; use crate::bindings::models; @@ -59,6 +59,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { workspace_commands::registration::register(module)?; auditing::registration::register(module)?; cursor::registration::register(module)?; + state_rows::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-state/Cargo.toml b/crates/sqlbuild-state/Cargo.toml new file mode 100644 index 0000000000..b8436c7922 --- /dev/null +++ b/crates/sqlbuild-state/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "sqlbuild-state" +version.workspace = true +description = "Append-only warehouse state rows: event IDs, typed event SQL and source freshness versions." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +sha2.workspace = true +sqlbuild-core.workspace = true +sqlbuild-cursor.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-state/src/events/_helpers/identity.rs b/crates/sqlbuild-state/src/events/_helpers/identity.rs new file mode 100644 index 0000000000..3a9cf11060 --- /dev/null +++ b/crates/sqlbuild-state/src/events/_helpers/identity.rs @@ -0,0 +1,49 @@ +//! Content-addressed event IDs: SHA-256 of a compact, ASCII-escaped JSON identity array. + +use sha2::{Digest, Sha256}; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::events::constants::{COMPACT_ITEM_SEPARATOR, COMPACT_KEY_SEPARATOR}; +use crate::events::models::EventRelation; + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +pub(crate) fn optional_text(value: Option<&str>) -> JsonValue { + value.map_or(JsonValue::Null, text) +} + +/// `MigrationRelation.identity`: lowercased database or null, schema or `""`, and name. +pub(crate) fn relation_identity(relation: &EventRelation) -> JsonValue { + JsonValue::Array(vec![ + optional_text( + relation + .database + .as_deref() + .map(str::to_lowercase) + .as_deref(), + ), + text( + &relation + .schema + .as_deref() + .unwrap_or_default() + .to_lowercase(), + ), + text(&relation.name.to_lowercase()), + ]) +} + +/// `sha256(json.dumps(value, separators=(",", ":"), ensure_ascii=True))` as hex. +pub(crate) fn json_digest(value: &JsonValue, sort_keys: bool) -> Result { + let options = StdlibJsonOptions { + sort_keys, + ..StdlibJsonOptions::new(None) + .with_separators(COMPACT_ITEM_SEPARATOR, COMPACT_KEY_SEPARATOR) + }; + let encoded: String = dumps(value, &JsonDialect::Stdlib(options))?; + Ok(format!("{:x}", Sha256::digest(encoded.as_bytes()))) +} diff --git a/crates/sqlbuild-state/src/events/_helpers/literals.rs b/crates/sqlbuild-state/src/events/_helpers/literals.rs new file mode 100644 index 0000000000..f5ddca242e --- /dev/null +++ b/crates/sqlbuild-state/src/events/_helpers/literals.rs @@ -0,0 +1,46 @@ +//! Typed SQL literals for state event columns, as `render_state_sql_literal` writes them. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::events::constants::{ + ESCAPED_QUOTE, NULL_LITERAL, QUOTE, STRING_VALUE_REQUIRED, TIMESTAMP_VALUE_REQUIRED, UTC_SUFFIX, +}; +use crate::events::errors::StateSqlError; +use crate::events::models::{StateColumnType, StateValue}; + +/// `value` in single quotes with embedded quotes doubled. +pub(crate) fn quoted(value: &str) -> String { + format!("{QUOTE}{}{QUOTE}", value.replace(QUOTE, ESCAPED_QUOTE)) +} + +/// `datetime.isoformat()` of a UTC timestamp. +pub(crate) fn utc_isoformat(local: &LocalDateTime) -> String { + let date = &local.date; + let mut text: String = format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", + date.year, date.month, date.day, local.hour, local.minute, local.second + ); + if local.microsecond != 0 { + text.push_str(&format!(".{:06}", local.microsecond)); + } + text.push_str(UTC_SUFFIX); + text +} + +/// One value as the literal its declared column type takes. +pub(crate) fn literal( + column_type: StateColumnType, + value: &StateValue, +) -> Result { + match (column_type, value) { + (_, StateValue::Null) => Ok(NULL_LITERAL.to_owned()), + (StateColumnType::String, StateValue::Text(text)) => Ok(quoted(text)), + (StateColumnType::String, _) => Err(StateSqlError(STRING_VALUE_REQUIRED.to_owned())), + (StateColumnType::TextTimestamp, StateValue::UtcTimestamp(local)) => { + Ok(quoted(&utc_isoformat(local))) + } + (StateColumnType::TextTimestamp, _) => { + Err(StateSqlError(TIMESTAMP_VALUE_REQUIRED.to_owned())) + } + } +} diff --git a/crates/sqlbuild-state/src/events/_helpers/mod.rs b/crates/sqlbuild-state/src/events/_helpers/mod.rs new file mode 100644 index 0000000000..059db07285 --- /dev/null +++ b/crates/sqlbuild-state/src/events/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod identity; +pub(crate) mod literals; diff --git a/crates/sqlbuild-state/src/events/constants.rs b/crates/sqlbuild-state/src/events/constants.rs new file mode 100644 index 0000000000..e6992b279b --- /dev/null +++ b/crates/sqlbuild-state/src/events/constants.rs @@ -0,0 +1,12 @@ +//! Identity tags and messages of append-only migration state events. + +pub(crate) const COLUMN_EVENT_TAG: &str = "column"; +pub(crate) const OLD_NAME_EVENT_TAG: &str = "old_name"; +pub(crate) const COMPACT_ITEM_SEPARATOR: &str = ","; +pub(crate) const COMPACT_KEY_SEPARATOR: &str = ":"; +pub(crate) const QUOTE: char = '\''; +pub(crate) const ESCAPED_QUOTE: &str = "''"; +pub(crate) const NULL_LITERAL: &str = "NULL"; +pub(crate) const UTC_SUFFIX: &str = "+00:00"; +pub const STRING_VALUE_REQUIRED: &str = "state string columns require str values"; +pub const TIMESTAMP_VALUE_REQUIRED: &str = "state timestamp columns require datetime values"; diff --git a/crates/sqlbuild-state/src/events/errors.rs b/crates/sqlbuild-state/src/events/errors.rs new file mode 100644 index 0000000000..0c60fe83bc --- /dev/null +++ b/crates/sqlbuild-state/src/events/errors.rs @@ -0,0 +1,15 @@ +//! Failures rendering one state event row. + +use std::fmt::{self, Display}; + +/// A value whose kind does not fit its declared state column, with Python's message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StateSqlError(pub String); + +impl Display for StateSqlError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for StateSqlError {} diff --git a/crates/sqlbuild-state/src/events/main/column_migration_event_id.rs b/crates/sqlbuild-state/src/events/main/column_migration_event_id.rs new file mode 100644 index 0000000000..012ff3d7f2 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/column_migration_event_id.rs @@ -0,0 +1,25 @@ +//! `column_migration_event_id`: the content-addressed ID of one column rename by one run. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::events::_helpers::identity::{json_digest, optional_text, relation_identity, text}; +use crate::events::constants::COLUMN_EVENT_TAG; +use crate::events::models::EventRelation; + +/// SHA-256 of `["column", run_id, target_name, relation identity, origin, destination]`. +pub fn column_migration_event_id( + (run_id, target_name): (&str, Option<&str>), + relation: &EventRelation, + (origin_column, destination_column): (&str, &str), +) -> Result { + let identity = JsonValue::Array(vec![ + text(COLUMN_EVENT_TAG), + text(run_id), + optional_text(target_name), + relation_identity(relation), + text(&origin_column.to_lowercase()), + text(&destination_column.to_lowercase()), + ]); + json_digest(&identity, false) +} diff --git a/crates/sqlbuild-state/src/events/main/event_exists_sql.rs b/crates/sqlbuild-state/src/events/main/event_exists_sql.rs new file mode 100644 index 0000000000..2afc579fac --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/event_exists_sql.rs @@ -0,0 +1,12 @@ +//! `event_exists_sql`: the lookup that makes an append-only event write idempotent. + +use crate::events::_helpers::literals::quoted; + +/// `SELECT event_id FROM WHERE event_id = ''`. +#[must_use] +pub fn event_exists_sql(table: &str, event_id: &str) -> String { + format!( + "SELECT event_id FROM {table} WHERE event_id = {}", + quoted(event_id) + ) +} diff --git a/crates/sqlbuild-state/src/events/main/event_insert_sql.rs b/crates/sqlbuild-state/src/events/main/event_insert_sql.rs new file mode 100644 index 0000000000..bd82422553 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/event_insert_sql.rs @@ -0,0 +1,19 @@ +//! `event_insert_sql`: one append-only event row as `INSERT ... VALUES` with typed literals. + +use crate::events::_helpers::literals::literal; +use crate::events::errors::StateSqlError; +use crate::events::models::StateCell; + +/// `INSERT INTO
() VALUES ()`, failing on the first ill-typed value. +pub fn event_insert_sql(table: &str, cells: &[StateCell]) -> Result { + let literals: Vec = cells + .iter() + .map(|cell| literal(cell.column_type, &cell.value)) + .collect::>()?; + let columns: Vec<&str> = cells.iter().map(|cell| cell.column.as_str()).collect(); + Ok(format!( + "INSERT INTO {table} ({}) VALUES ({})", + columns.join(", "), + literals.join(", ") + )) +} diff --git a/crates/sqlbuild-state/src/events/main/migration_event_id.rs b/crates/sqlbuild-state/src/events/main/migration_event_id.rs new file mode 100644 index 0000000000..325d2fca7c --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/migration_event_id.rs @@ -0,0 +1,22 @@ +//! `migration_event_id`: the content-addressed ID of one relation move performed by one run. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::events::_helpers::identity::{json_digest, optional_text, relation_identity, text}; +use crate::events::models::EventRelation; + +/// SHA-256 of `[run_id, target_name, origin identity, destination identity]`. +pub fn migration_event_id( + (run_id, target_name): (&str, Option<&str>), + origin: &EventRelation, + destination: &EventRelation, +) -> Result { + let identity = JsonValue::Array(vec![ + text(run_id), + optional_text(target_name), + relation_identity(origin), + relation_identity(destination), + ]); + json_digest(&identity, false) +} diff --git a/crates/sqlbuild-state/src/events/main/mod.rs b/crates/sqlbuild-state/src/events/main/mod.rs new file mode 100644 index 0000000000..3870be1589 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/mod.rs @@ -0,0 +1,5 @@ +pub mod column_migration_event_id; +pub mod event_exists_sql; +pub mod event_insert_sql; +pub mod migration_event_id; +pub mod old_name_view_event_id; diff --git a/crates/sqlbuild-state/src/events/main/old_name_view_event_id.rs b/crates/sqlbuild-state/src/events/main/old_name_view_event_id.rs new file mode 100644 index 0000000000..785fd45008 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/old_name_view_event_id.rs @@ -0,0 +1,20 @@ +//! `old_name_view_event_id`: the content-addressed ID of one old-name step of a recorded move. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::events::_helpers::identity::{json_digest, text}; +use crate::events::constants::OLD_NAME_EVENT_TAG; + +/// SHA-256 of `["old_name", event_type, migration_event_id]`. +pub fn old_name_view_event_id( + event_type: &str, + migration_event_id: &str, +) -> Result { + let identity = JsonValue::Array(vec![ + text(OLD_NAME_EVENT_TAG), + text(event_type), + text(migration_event_id), + ]); + json_digest(&identity, false) +} diff --git a/crates/sqlbuild-state/src/events/mod.rs b/crates/sqlbuild-state/src/events/mod.rs new file mode 100644 index 0000000000..899237ebcb --- /dev/null +++ b/crates/sqlbuild-state/src/events/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/events/models.rs b/crates/sqlbuild-state/src/events/models.rs new file mode 100644 index 0000000000..9f850af603 --- /dev/null +++ b/crates/sqlbuild-state/src/events/models.rs @@ -0,0 +1,37 @@ +//! Relations, typed state values and column types of migration state events. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +/// A relation named by a migration event; only the name is required. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EventRelation { + pub database: Option, + pub schema: Option, + pub name: String, +} + +/// The declared physical type of a state event column. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StateColumnType { + String, + TextTimestamp, +} + +/// One value of an event row as the caller holds it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StateValue { + Null, + Text(String), + /// A timestamp already normalized to UTC wall time. + UtcTimestamp(LocalDateTime), + /// Any other value, which no supported column type accepts. + Other, +} + +/// One column of an event row: its name, declared type and value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StateCell { + pub column: String, + pub column_type: StateColumnType, + pub value: StateValue, +} diff --git a/crates/sqlbuild-state/src/events/tests.rs b/crates/sqlbuild-state/src/events/tests.rs new file mode 100644 index 0000000000..b55f12f730 --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_insert_sql.rs"] +mod insert_sql; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/events/tests/helpers.rs b/crates/sqlbuild-state/src/events/tests/helpers.rs new file mode 100644 index 0000000000..3ad5209a10 --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/helpers.rs @@ -0,0 +1,9 @@ +use crate::events::models::{StateCell, StateColumnType, StateValue}; + +pub(crate) fn cell(column: &str, column_type: StateColumnType, value: StateValue) -> StateCell { + StateCell { + column: column.to_owned(), + column_type, + value, + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_insert_sql.rs b/crates/sqlbuild-state/src/events/tests/test_insert_sql.rs new file mode 100644 index 0000000000..6da33933b0 --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/test_insert_sql.rs @@ -0,0 +1,47 @@ +use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; + +use crate::events::errors::StateSqlError; +use crate::events::main::event_insert_sql::event_insert_sql; +use crate::events::models::{StateColumnType, StateValue}; +use crate::events::tests::helpers::cell; +use crate::events::tests::test_types::InsertTestCase; + +#[test] +fn given_event_row_when_rendering_insert_then_matches_python_literals() { + let created = LocalDateTime { + date: CivilDate { + year: 2024, + month: 2, + day: 29, + }, + hour: 1, + minute: 2, + second: 3, + microsecond: 40, + }; + let test_cases = [ + InsertTestCase { + description: "quoted text, a null and a UTC timestamp", + cells: vec![ + cell("event_id", StateColumnType::String, StateValue::Text("o'k".to_owned())), + cell("target_name", StateColumnType::String, StateValue::Null), + cell("created_at", StateColumnType::TextTimestamp, StateValue::UtcTimestamp(created)), + ], + expected_sql: Ok("INSERT INTO s.t (event_id, target_name, created_at) VALUES ('o''k', NULL, '2024-02-29T01:02:03.000040+00:00')".to_owned()), + }, + InsertTestCase { + description: "a non-text value in a string column", + cells: vec![cell("event_id", StateColumnType::String, StateValue::Other)], + expected_sql: Err(StateSqlError("state string columns require str values".to_owned())), + }, + ]; + + for test_case in test_cases { + assert_eq!( + event_insert_sql("s.t", &test_case.cells), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_types.rs b/crates/sqlbuild-state/src/events/tests/test_types.rs new file mode 100644 index 0000000000..46f300b779 --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::events::errors::StateSqlError; +use crate::events::models::StateCell; + +pub(crate) struct InsertTestCase { + pub(crate) description: &'static str, + pub(crate) cells: Vec, + pub(crate) expected_sql: Result, +} diff --git a/crates/sqlbuild-state/src/freshness/_helpers/durations.rs b/crates/sqlbuild-state/src/freshness/_helpers/durations.rs new file mode 100644 index 0000000000..f792162144 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/_helpers/durations.rs @@ -0,0 +1,34 @@ +//! Single-unit minute, hour or day durations, as `Duration.parse` and `fixed_seconds` read them. + +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_cursor::algebra::models::DurationDigits; + +use crate::freshness::constants::{SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE}; + +/// Seconds of a duration; `None` inside `Some` when they exceed `u128`. +pub(crate) type DurationSeconds = Option; + +fn is_zero(digits: &str) -> bool { + digits.bytes().all(|byte| byte == b'0') +} + +/// The seconds of `text` when it names exactly one of minutes, hours or days. +pub(crate) fn single_unit_seconds(text: &str) -> Option { + let digits: DurationDigits = parse_duration(text)?; + let fixed: [(&str, u128); 3] = [ + (&digits.days, SECONDS_PER_DAY), + (&digits.hours, SECONDS_PER_HOUR), + (&digits.minutes, SECONDS_PER_MINUTE), + ]; + let calendar_or_seconds: bool = + !is_zero(&digits.years) || !is_zero(&digits.months) || !is_zero(&digits.seconds); + let mut used = fixed.iter().filter(|(amount, _)| !is_zero(amount)); + let (amount, unit) = used.next()?; + if calendar_or_seconds || used.next().is_some() { + return None; + } + let Ok(value) = amount.parse::() else { + return Some(None); + }; + Some(value.checked_mul(*unit)) +} diff --git a/crates/sqlbuild-state/src/freshness/_helpers/mod.rs b/crates/sqlbuild-state/src/freshness/_helpers/mod.rs new file mode 100644 index 0000000000..3fc35392a0 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod durations; +pub(crate) mod moments; diff --git a/crates/sqlbuild-state/src/freshness/_helpers/moments.rs b/crates/sqlbuild-state/src/freshness/_helpers/moments.rs new file mode 100644 index 0000000000..78aca6613a --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/_helpers/moments.rs @@ -0,0 +1,42 @@ +//! Microsecond distances between UTC wall times. + +use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; + +use crate::freshness::constants::{ + DAYS_PER_ERA, DAYS_PER_YEAR, ERA_YEARS, FEBRUARY, MARCH_BASED_MONTH_DAYS, MICROS_PER_SECOND, + MONTH_LENGTH_DIVISOR, MONTHS_FROM_MARCH, SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE, +}; + +/// Days since 0000-03-01 in the proleptic Gregorian calendar. +fn day_number(date: &CivilDate) -> i128 { + let shifted: bool = date.month <= FEBRUARY; + let year: i128 = i128::from(date.year) - i128::from(shifted); + let era: i128 = year.div_euclid(ERA_YEARS); + let year_of_era: i128 = year - era * ERA_YEARS; + let month: i128 = i128::from(date.month); + let month_from_march: i128 = if shifted { + month + MONTHS_FROM_MARCH + } else { + month - 3 + }; + let day_of_year: i128 = (MARCH_BASED_MONTH_DAYS * month_from_march + 2) / MONTH_LENGTH_DIVISOR + + i128::from(date.day) + - 1; + let day_of_era: i128 = + year_of_era * DAYS_PER_YEAR + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * DAYS_PER_ERA + day_of_era +} + +fn micros_of_day(local: &LocalDateTime) -> i128 { + let seconds: u128 = u128::from(local.hour) * SECONDS_PER_HOUR + + u128::from(local.minute) * SECONDS_PER_MINUTE + + u128::from(local.second); + (seconds * MICROS_PER_SECOND + u128::from(local.microsecond)) as i128 +} + +/// `later - earlier` in microseconds, as subtracting two `datetime` values gives it. +pub(crate) fn elapsed_micros(earlier: &LocalDateTime, later: &LocalDateTime) -> i128 { + let days: i128 = day_number(&later.date) - day_number(&earlier.date); + days * (SECONDS_PER_DAY * MICROS_PER_SECOND) as i128 + micros_of_day(later) + - micros_of_day(earlier) +} diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs new file mode 100644 index 0000000000..280fa1be44 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -0,0 +1,30 @@ +//! Value kinds, payload keys, duration units and messages of source freshness state. + +pub const TIMESTAMP_VALUE_KIND: &str = "timestamp"; +pub(crate) const SOURCE_NAME_KEY: &str = "source_name"; +pub(crate) const STRATEGY_KEY: &str = "strategy"; +pub(crate) const VALUE_KIND_KEY: &str = "value_kind"; +pub(crate) const DATA_VERSION_KEY: &str = "data_version"; +pub(crate) const COMPACT_ITEM_SEPARATOR: &str = ","; +pub(crate) const COMPACT_KEY_SEPARATOR: &str = ":"; +pub(crate) const UTC_SUFFIX: &str = "+00:00"; +pub(crate) const SECONDS_PER_MINUTE: u128 = 60; +pub(crate) const SECONDS_PER_HOUR: u128 = 3_600; +pub(crate) const SECONDS_PER_DAY: u128 = 86_400; +pub(crate) const MICROS_PER_SECOND: u128 = 1_000_000; +/// `timedelta.max.days`. +pub(crate) const MAX_TIMEDELTA_DAYS: u128 = 999_999_999; +/// The largest C `int` a `timedelta` day count passes through. +pub(crate) const MAX_C_INT: u128 = i32::MAX as u128; +pub(crate) const C_INT_OVERFLOW: &str = "Python int too large to convert to C int"; +pub const LAG_TOLERANCE_INVALID: &str = + "source freshness lag_tolerance must be a positive duration like 15m, 2h, or 1d"; +pub const AGE_POLICY_INVALID: &str = + "source freshness age_policy value must be a positive duration like 15m, 2h, or 1d"; +pub(crate) const ERA_YEARS: i128 = 400; +pub(crate) const DAYS_PER_ERA: i128 = 146_097; +pub(crate) const DAYS_PER_YEAR: i128 = 365; +pub(crate) const MARCH_BASED_MONTH_DAYS: i128 = 153; +pub(crate) const MONTH_LENGTH_DIVISOR: i128 = 5; +pub(crate) const MONTHS_FROM_MARCH: i128 = 9; +pub(crate) const FEBRUARY: u32 = 2; diff --git a/crates/sqlbuild-state/src/freshness/errors.rs b/crates/sqlbuild-state/src/freshness/errors.rs new file mode 100644 index 0000000000..b85513cbdb --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/errors.rs @@ -0,0 +1,24 @@ +//! Source freshness failures, each the exception Python raises for the same input. + +use std::fmt::{self, Display}; + +/// A failed freshness decision and the Python exception class that reports it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FreshnessError { + /// `SourceFreshnessObservationError`. + Observation(String), + /// `SourceFreshnessInputError`. + Input(String), + /// `OverflowError` from building a `timedelta`. + Overflow(String), +} + +impl Display for FreshnessError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Observation(message) | Self::Input(message) | Self::Overflow(message) => message, + }) + } +} + +impl std::error::Error for FreshnessError {} diff --git a/crates/sqlbuild-state/src/freshness/main/age_status.rs b/crates/sqlbuild-state/src/freshness/main/age_status.rs new file mode 100644 index 0000000000..7e51eef2b0 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/age_status.rs @@ -0,0 +1,41 @@ +//! `age_status`: an observed timestamp's age against its warn and error thresholds. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::freshness::_helpers::durations::single_unit_seconds; +use crate::freshness::_helpers::moments::elapsed_micros; +use crate::freshness::constants::{AGE_POLICY_INVALID, MICROS_PER_SECOND}; +use crate::freshness::errors::FreshnessError; +use crate::freshness::models::AgeStatus; + +/// Whether `age_seconds > threshold`, as Python compares a float with an integer. +fn older_than(age_micros: i128, threshold: &str) -> Result { + let Some(seconds) = single_unit_seconds(threshold) else { + return Err(FreshnessError::Input(AGE_POLICY_INVALID.to_owned())); + }; + let Some(seconds) = seconds else { + return Ok(false); + }; + let age_seconds: f64 = age_micros as f64 / MICROS_PER_SECOND as f64; + Ok(age_seconds > seconds as f64) +} + +/// The data version's age at `observed_at`: `error` past `error_after`, else `warn` past `warn_after`, else `pass`. +pub fn age_status( + (data_version, observed_at): (&LocalDateTime, &LocalDateTime), + error_after: Option<&str>, + warn_after: Option<&str>, +) -> Result { + let age_micros: i128 = elapsed_micros(data_version, observed_at); + if let Some(threshold) = error_after + && older_than(age_micros, threshold)? + { + return Ok(AgeStatus::Error); + } + if let Some(threshold) = warn_after + && older_than(age_micros, threshold)? + { + return Ok(AgeStatus::Warn); + } + Ok(AgeStatus::Pass) +} diff --git a/crates/sqlbuild-state/src/freshness/main/data_version_hash.rs b/crates/sqlbuild-state/src/freshness/main/data_version_hash.rs new file mode 100644 index 0000000000..4fd854c9aa --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/data_version_hash.rs @@ -0,0 +1,37 @@ +//! `data_version_hash`: the stable source freshness identity used as a graph input. + +use sha2::{Digest, Sha256}; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::freshness::constants::{ + COMPACT_ITEM_SEPARATOR, COMPACT_KEY_SEPARATOR, DATA_VERSION_KEY, SOURCE_NAME_KEY, STRATEGY_KEY, + VALUE_KIND_KEY, +}; + +/// SHA-256 of the compact, key-sorted JSON of the four identity fields. +pub fn data_version_hash( + (source_name, strategy): (&str, &str), + value_kind: &str, + data_version: &str, +) -> Result { + let payload = JsonValue::Object( + [ + (SOURCE_NAME_KEY, source_name), + (STRATEGY_KEY, strategy), + (VALUE_KIND_KEY, value_kind), + (DATA_VERSION_KEY, data_version), + ] + .into_iter() + .map(|(key, value)| (key.to_owned(), JsonValue::String(value.to_owned()))) + .collect(), + ); + let options = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None) + .with_separators(COMPACT_ITEM_SEPARATOR, COMPACT_KEY_SEPARATOR) + }; + let encoded: String = dumps(&payload, &JsonDialect::Stdlib(options))?; + Ok(format!("{:x}", Sha256::digest(encoded.as_bytes()))) +} diff --git a/crates/sqlbuild-state/src/freshness/main/mod.rs b/crates/sqlbuild-state/src/freshness/main/mod.rs new file mode 100644 index 0000000000..6e7cddd828 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/mod.rs @@ -0,0 +1,4 @@ +pub mod age_status; +pub mod data_version_hash; +pub mod normalized_data_version; +pub mod records_equivalent; diff --git a/crates/sqlbuild-state/src/freshness/main/normalized_data_version.rs b/crates/sqlbuild-state/src/freshness/main/normalized_data_version.rs new file mode 100644 index 0000000000..306ef41e8e --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/normalized_data_version.rs @@ -0,0 +1,25 @@ +//! `normalized_data_version`: a data version as stable state text. + +use crate::freshness::constants::UTC_SUFFIX; +use crate::freshness::models::DataVersion; + +/// A timestamp's `isoformat()`, an integer's digits, or the text itself. +#[must_use] +pub fn normalized_data_version(value: &DataVersion) -> String { + let (local, utc) = match value { + DataVersion::Timestamp { local, utc } => (local, *utc), + DataVersion::Integer(text) | DataVersion::Text(text) => return text.clone(), + }; + let date = &local.date; + let mut text: String = format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", + date.year, date.month, date.day, local.hour, local.minute, local.second + ); + if local.microsecond != 0 { + text.push_str(&format!(".{:06}", local.microsecond)); + } + if utc { + text.push_str(UTC_SUFFIX); + } + text +} diff --git a/crates/sqlbuild-state/src/freshness/main/records_equivalent.rs b/crates/sqlbuild-state/src/freshness/main/records_equivalent.rs new file mode 100644 index 0000000000..8b308c9da5 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/records_equivalent.rs @@ -0,0 +1,63 @@ +//! `records_equivalent`: whether two freshness records let a source be skipped. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::freshness::_helpers::durations::single_unit_seconds; +use crate::freshness::_helpers::moments::elapsed_micros; +use crate::freshness::constants::{ + C_INT_OVERFLOW, LAG_TOLERANCE_INVALID, MAX_C_INT, MAX_TIMEDELTA_DAYS, MICROS_PER_SECOND, + SECONDS_PER_DAY, TIMESTAMP_VALUE_KIND, +}; +use crate::freshness::errors::FreshnessError; +use crate::freshness::models::ComparableRecord; +use crate::freshness::types::TimestampParser; + +/// `timedelta(seconds=...)` of a lag tolerance in microseconds, failing as Python does. +fn tolerance_micros(text: &str) -> Result { + let Some(seconds) = single_unit_seconds(text) else { + return Err(FreshnessError::Observation( + LAG_TOLERANCE_INVALID.to_owned(), + )); + }; + let days: Option = seconds.map(|seconds| seconds / SECONDS_PER_DAY); + match (seconds, days) { + (Some(seconds), Some(days)) if days <= MAX_TIMEDELTA_DAYS => { + Ok(seconds * MICROS_PER_SECOND) + } + (_, Some(days)) if days <= MAX_C_INT => Err(FreshnessError::Overflow(format!( + "days={days}; must have magnitude <= {MAX_TIMEDELTA_DAYS}" + ))), + _ => Err(FreshnessError::Overflow(C_INT_OVERFLOW.to_owned())), + } +} + +/// Equal hashes, or equal timestamps, or a later timestamp within `lag_tolerance`. +pub fn records_equivalent( + (previous, current): (&ComparableRecord, &ComparableRecord), + lag_tolerance: Option<&str>, + parser: &dyn TimestampParser, +) -> Result { + if previous.data_version_hash == current.data_version_hash { + return Ok(true); + } + if previous.value_kind != TIMESTAMP_VALUE_KIND || current.value_kind != TIMESTAMP_VALUE_KIND { + return Ok(false); + } + let (Some(previous_text), Some(current_text)) = (&previous.data_version, ¤t.data_version) + else { + return Ok(false); + }; + let previous_time: LocalDateTime = parser.utc_timestamp(previous_text)?; + let current_time: LocalDateTime = parser.utc_timestamp(current_text)?; + let elapsed: i128 = elapsed_micros(&previous_time, ¤t_time); + if elapsed == 0 { + return Ok(true); + } + let Some(tolerance) = lag_tolerance else { + return Ok(false); + }; + if elapsed < 0 { + return Ok(false); + } + Ok(elapsed.unsigned_abs() <= tolerance_micros(tolerance)?) +} diff --git a/crates/sqlbuild-state/src/freshness/mod.rs b/crates/sqlbuild-state/src/freshness/mod.rs new file mode 100644 index 0000000000..65aaca2009 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/freshness/models.rs b/crates/sqlbuild-state/src/freshness/models.rs new file mode 100644 index 0000000000..99ca646dfe --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/models.rs @@ -0,0 +1,32 @@ +//! Observed data versions, comparable state records and age statuses. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +/// An observed data version of one value kind. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DataVersion { + /// A timestamp's wall time; `utc` when it was aware and is now in UTC. + Timestamp { + local: LocalDateTime, + utc: bool, + }, + /// An integer's decimal text. + Integer(String), + Text(String), +} + +/// The fields of a stored or observed record that skip decisions compare. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ComparableRecord { + pub data_version_hash: String, + pub value_kind: String, + pub data_version: Option, +} + +/// How an observed timestamp's age compares with its policy thresholds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AgeStatus { + Pass, + Warn, + Error, +} diff --git a/crates/sqlbuild-state/src/freshness/tests.rs b/crates/sqlbuild-state/src/freshness/tests.rs new file mode 100644 index 0000000000..ead02b3221 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_equivalence.rs"] +mod equivalence; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/freshness/tests/helpers.rs b/crates/sqlbuild-state/src/freshness/tests/helpers.rs new file mode 100644 index 0000000000..ac0bb6bd79 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/tests/helpers.rs @@ -0,0 +1,31 @@ +use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; + +use crate::freshness::errors::FreshnessError; +use crate::freshness::models::ComparableRecord; +use crate::freshness::tests::test_types::MinuteParser; +use crate::freshness::types::TimestampParser; + +pub(crate) fn record(data_version: &str) -> ComparableRecord { + ComparableRecord { + data_version_hash: data_version.to_owned(), + value_kind: "timestamp".to_owned(), + data_version: Some(data_version.to_owned()), + } +} + +impl TimestampParser for MinuteParser { + fn utc_timestamp(&self, text: &str) -> Result { + let field = |range: std::ops::Range| text[range].parse::().unwrap_or(0); + Ok(LocalDateTime { + date: CivilDate { + year: i64::from(field(0..4)), + month: field(5..7), + day: field(8..10), + }, + hour: field(11..13), + minute: field(14..16), + second: 0, + microsecond: 0, + }) + } +} diff --git a/crates/sqlbuild-state/src/freshness/tests/test_equivalence.rs b/crates/sqlbuild-state/src/freshness/tests/test_equivalence.rs new file mode 100644 index 0000000000..c78af54e91 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/tests/test_equivalence.rs @@ -0,0 +1,56 @@ +use crate::freshness::errors::FreshnessError; +use crate::freshness::main::records_equivalent::records_equivalent; +use crate::freshness::tests::helpers::record; +use crate::freshness::tests::test_types::{EquivalenceTestCase, MinuteParser}; + +#[test] +fn given_timestamp_records_when_comparing_then_lag_tolerance_decides() { + let test_cases = [ + EquivalenceTestCase { + description: "later within a two-hour tolerance across a month end", + previous: "2024-01-31T23:30", + current: "2024-02-01T01:30", + lag_tolerance: Some("2h"), + expected_equivalent: Ok(true), + }, + EquivalenceTestCase { + description: "later beyond a fifteen-minute tolerance", + previous: "2024-01-31T23:30", + current: "2024-01-31T23:46", + lag_tolerance: Some("15m"), + expected_equivalent: Ok(false), + }, + EquivalenceTestCase { + description: "an earlier value is never equivalent", + previous: "2024-01-31T23:30", + current: "2024-01-31T23:29", + lag_tolerance: Some("1d"), + expected_equivalent: Ok(false), + }, + EquivalenceTestCase { + description: "a calendar tolerance is rejected", + previous: "2024-01-31T23:30", + current: "2024-01-31T23:31", + lag_tolerance: Some("1mo"), + expected_equivalent: Err(FreshnessError::Observation( + "source freshness lag_tolerance must be a positive duration like 15m, 2h, or 1d" + .to_owned(), + )), + }, + ]; + + for test_case in test_cases { + let (previous, current) = (record(test_case.previous), record(test_case.current)); + + assert_eq!( + records_equivalent( + (&previous, ¤t), + test_case.lag_tolerance, + &MinuteParser + ), + test_case.expected_equivalent, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/freshness/tests/test_types.rs b/crates/sqlbuild-state/src/freshness/tests/test_types.rs new file mode 100644 index 0000000000..0632d8083a --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/tests/test_types.rs @@ -0,0 +1,12 @@ +use crate::freshness::errors::FreshnessError; + +/// Parses `YYYY-MM-DDTHH:MM` text the tests write. +pub(crate) struct MinuteParser; + +pub(crate) struct EquivalenceTestCase { + pub(crate) description: &'static str, + pub(crate) previous: &'static str, + pub(crate) current: &'static str, + pub(crate) lag_tolerance: Option<&'static str>, + pub(crate) expected_equivalent: Result, +} diff --git a/crates/sqlbuild-state/src/freshness/types.rs b/crates/sqlbuild-state/src/freshness/types.rs new file mode 100644 index 0000000000..ed64105016 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/types.rs @@ -0,0 +1,11 @@ +//! The timestamp parsing a caller supplies for stored data versions. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::freshness::errors::FreshnessError; + +/// Parses stored timestamp text into UTC wall time. +pub trait TimestampParser { + /// `datetime.fromisoformat(text)` in UTC, naive text read as UTC. + fn utc_timestamp(&self, text: &str) -> Result; +} diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs new file mode 100644 index 0000000000..d62c133a7e --- /dev/null +++ b/crates/sqlbuild-state/src/lib.rs @@ -0,0 +1,6 @@ +//! Warehouse state rows SQLBuild appends: event IDs, event SQL and source freshness versions. + +#![forbid(unsafe_code)] + +pub mod events; +pub mod freshness; diff --git a/fensu.toml b/fensu.toml index 046d307047..bec5e7a491 100644 --- a/fensu.toml +++ b/fensu.toml @@ -369,6 +369,7 @@ crate_names = [ "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", + "sqlbuild-state", ] [[dupes.contract_exemptions]] diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f37f91ff5d..22c21490f5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -140,6 +140,8 @@ from sqlbuild.compiler.lineage.models import ( ModelColumnLineage, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability +from sqlbuild.compiler.migrations.models import MigrationRelation +from sqlbuild.compiler.migrations.types import OldNameViewEventType from sqlbuild.compiler.pipeline.models import ( CompiledProjectPhaseResult, CompilePipelineOptions, @@ -194,6 +196,10 @@ from sqlbuild.compiler.scopes.models import ( ) from sqlbuild.compiler.scopes.types import JsonValue, ScopeQueryTarget from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord +from sqlbuild.compiler.source_freshness.types import ( + SourceFreshnessAgeStatus, + SourceFreshnessComparableRecord, +) from sqlbuild.compiler.sql_analysis.models import ( SqlBindingResult, SqlLexicalSyntax, @@ -242,11 +248,17 @@ from sqlbuild.spec.contracts.models import ( SchemaSeedEntry, SettingsConfig, SourceEntry, + SourceFreshnessAgePolicy, SourceLocation, TargetConfig, ) -from sqlbuild.spec.contracts.types import WarehouseGroup +from sqlbuild.spec.contracts.types import ( + SourceFreshnessStrategy, + SourceFreshnessValueKind, + WarehouseGroup, +) from sqlbuild.sql_values.models import SqlValue +from sqlbuild.sql_values.types import StateSqlValueType BUILD_IDENTITY: str STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] @@ -2906,6 +2918,54 @@ def _oracle_yaml_load(text: str) -> str: ... def _oracle_toml_load(text: str) -> str: ... # Discovery: header values and authored locations. +def deterministic_migration_event_id( + *, + run_id: str, + target_name: str | None, + origin: MigrationRelation, + destination: MigrationRelation, +) -> str: ... +def deterministic_column_migration_event_id( + *, + run_id: str, + target_name: str | None, + relation: MigrationRelation, + origin_column: str, + destination_column: str, +) -> str: ... +def deterministic_old_name_view_event_id( + *, event_type: OldNameViewEventType, migration_event_id: str +) -> str: ... +def render_event_exists_sql(*, table: str, event_id: str) -> str: ... +def render_event_insert_sql( + *, + table: str, + columns: tuple[str, ...], + column_types: Mapping[str, StateSqlValueType], + values: tuple[object | None, ...], +) -> str: ... +def source_freshness_data_version_hash( + *, + source_name: str, + strategy: SourceFreshnessStrategy | str, + value_kind: SourceFreshnessValueKind | str, + data_version: str, +) -> str: ... +def normalize_source_freshness_data_version( + *, value: object, value_kind: SourceFreshnessValueKind +) -> str: ... +def source_freshness_records_equivalent( + *, + previous_record: SourceFreshnessComparableRecord, + current_record: SourceFreshnessComparableRecord, + lag_tolerance: str | None = None, +) -> bool: ... +def evaluate_source_freshness_age_policy( + *, + policy: SourceFreshnessAgePolicy | None, + data_version: object, + observed_at: datetime, +) -> SourceFreshnessAgeStatus | None: ... def cursor_parse(*, raw: object, cursor_type: str) -> CursorScalar: ... def cursor_try_parse(*, raw: object, cursor_type: str) -> CursorScalar | None: ... def cursor_render(*, value: CursorScalar) -> str: ... diff --git a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py b/src/sqlbuild/compiler/migrations/_helpers/column_sql.py index f70055039d..4cacd717c9 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/column_sql.py @@ -5,6 +5,7 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import render_event_exists_sql, render_event_insert_sql from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -12,8 +13,6 @@ from sqlbuild.compiler.migrations._helpers.sql import ( decode_event_timestamp, optional_text, - render_event_exists_sql, - render_event_insert_sql, ) from sqlbuild.compiler.migrations.constants import ( COLUMN_MIGRATION_COLUMN_TYPES, diff --git a/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py b/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py index f14967b3a1..0f9e9b4887 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py @@ -6,6 +6,7 @@ from collections.abc import Callable, Iterable from typing import Any +from sqlbuild._native import render_event_exists_sql, render_event_insert_sql from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -13,8 +14,6 @@ from sqlbuild.compiler.migrations._helpers.sql import ( decode_event_timestamp, optional_text, - render_event_exists_sql, - render_event_insert_sql, ) from sqlbuild.compiler.migrations.constants import ( OLD_NAME_VIEW_COLUMN_TYPES, diff --git a/src/sqlbuild/compiler/migrations/_helpers/sql.py b/src/sqlbuild/compiler/migrations/_helpers/sql.py index 9773f4a003..0cf1616b6d 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/sql.py @@ -2,10 +2,11 @@ from __future__ import annotations -from collections.abc import Callable, Mapping +from collections.abc import Callable from datetime import UTC, datetime from typing import Any +from sqlbuild._native import render_event_exists_sql, render_event_insert_sql from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -18,8 +19,6 @@ from sqlbuild.compiler.migrations.exceptions import MigrationStateError from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal -from sqlbuild.sql_values.types import StateSqlValueType def qualified_migration_table( @@ -70,31 +69,6 @@ def build_insert_sql( ) -def render_event_exists_sql(*, table: str, event_id: str) -> str: - """Render the lookup that makes an append-only event write idempotent.""" - - event_id_literal: str = render_state_sql_literal( - value=event_id, declared_type=StateSqlValueType.STRING - ) - return f"SELECT event_id FROM {table} WHERE event_id = {event_id_literal}" - - -def render_event_insert_sql( - *, - table: str, - columns: tuple[str, ...], - column_types: Mapping[str, StateSqlValueType], - values: tuple[object | None, ...], -) -> str: - """Render one append-only event row as INSERT ... VALUES with typed literals.""" - - literals: str = ", ".join( - render_state_sql_literal(value=value, declared_type=column_types[column]) - for column, value in zip(columns, values, strict=True) - ) - return f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({literals})" - - def _event_table(*, event: MigrationEvent, render_qualified_name: Callable[..., str | None]) -> str: if event.destination.schema is None: raise MigrationStateError("model migration events require a destination schema") diff --git a/src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py b/src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py deleted file mode 100644 index 9f7882154b..0000000000 --- a/src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Build stable identifiers for immutable column migration events.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.compiler.migrations.models import MigrationRelation - - -def deterministic_column_migration_event_id( - *, - run_id: str, - target_name: str | None, - relation: MigrationRelation, - origin_column: str, - destination_column: str, -) -> str: - """Return the content-addressed ID for one column rename performed by one run.""" - - identity: tuple[object, ...] = ( - "column", - run_id, - target_name, - relation.identity, - origin_column.lower(), - destination_column.lower(), - ) - encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py b/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py deleted file mode 100644 index 7c9ecf2f68..0000000000 --- a/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Build stable identifiers for immutable model migration events.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.compiler.migrations.models import MigrationRelation - - -def deterministic_migration_event_id( - *, - run_id: str, - target_name: str | None, - origin: MigrationRelation, - destination: MigrationRelation, -) -> str: - """Return the content-addressed ID for one migration performed by one run.""" - - identity: tuple[object, ...] = ( - run_id, - target_name, - origin.identity, - destination.identity, - ) - encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/compiler/migrations/main/deterministic_old_name_view_event_id.py b/src/sqlbuild/compiler/migrations/main/deterministic_old_name_view_event_id.py deleted file mode 100644 index c1e36b912d..0000000000 --- a/src/sqlbuild/compiler/migrations/main/deterministic_old_name_view_event_id.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Build stable identifiers for immutable old-name view facts.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.compiler.migrations.types import OldNameViewEventType - - -def deterministic_old_name_view_event_id( - *, event_type: OldNameViewEventType, migration_event_id: str -) -> str: - """Return the content-addressed ID of one old-name step of one recorded move.""" - - identity: tuple[object, ...] = ("old_name", event_type.value, migration_event_id) - encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/age_policy.py b/src/sqlbuild/compiler/source_freshness/_helpers/age_policy.py deleted file mode 100644 index 91bcab43e4..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/age_policy.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Source freshness age policy evaluation helpers.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild.compiler.source_freshness._helpers.datetime import require_aware_utc_datetime -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessInputError -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus -from sqlbuild.cursor_algebra.constants import MINUTE_TO_DAY_DURATION_UNITS -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy - - -def evaluate_source_freshness_age_policy( - *, - policy: SourceFreshnessAgePolicy | None, - data_version: object, - observed_at: datetime, -) -> SourceFreshnessAgeStatus | None: - """Evaluate optional timestamp age policy for one source freshness observation.""" - - if policy is None: - return None - if not isinstance(data_version, datetime): - return SourceFreshnessAgeStatus.UNKNOWN - aware_data_version: datetime = require_aware_utc_datetime( - value=data_version, field_name="data_version" - ) - aware_observed_at: datetime = require_aware_utc_datetime( - value=observed_at, field_name="observed_at" - ) - age_seconds: float = (aware_observed_at - aware_data_version).total_seconds() - if policy.error_after is not None and age_seconds > _duration_seconds(policy.error_after): - return SourceFreshnessAgeStatus.ERROR - if policy.warn_after is not None and age_seconds > _duration_seconds(policy.warn_after): - return SourceFreshnessAgeStatus.WARN - return SourceFreshnessAgeStatus.PASS - - -def _duration_seconds(value: str) -> int: - duration: Duration | None = Duration.parse(value) - if duration is None or not duration.is_single_unit_in(MINUTE_TO_DAY_DURATION_UNITS): - raise SourceFreshnessInputError( - "source freshness age_policy value must be a positive duration like 15m, 2h, or 1d" - ) - return duration.fixed_seconds diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/planning.py b/src/sqlbuild/compiler/source_freshness/_helpers/planning.py index eb530d0af4..3a29acfe20 100644 --- a/src/sqlbuild/compiler/source_freshness/_helpers/planning.py +++ b/src/sqlbuild/compiler/source_freshness/_helpers/planning.py @@ -6,11 +6,14 @@ from datetime import datetime from typing import Any -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.compiler.source_freshness._helpers.age_policy import ( +from sqlbuild._native import ( evaluate_source_freshness_age_policy, + normalize_source_freshness_data_version, + source_freshness_data_version_hash, + source_freshness_records_equivalent, ) +from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter +from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.compiler.source_freshness.constants import ( INCOMPLETE_CONFIGURATION_ERROR_FRAGMENT, PHYSICAL_TABLE_SOURCE_ERROR_FRAGMENT, @@ -19,19 +22,10 @@ from sqlbuild.compiler.source_freshness.main.adapter_observation import ( observe_adapter_sources_freshness, ) -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) from sqlbuild.compiler.source_freshness.main.observation import ( observe_configured_source_freshness, ) from sqlbuild.compiler.source_freshness.main.read import read_latest_source_freshness -from sqlbuild.compiler.source_freshness.main.record_equivalence import ( - source_freshness_records_equivalent, -) from sqlbuild.compiler.source_freshness.models import ( AdapterSourceFreshnessBatch, DirectSourceFreshnessPlanningResult, diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/state.py b/src/sqlbuild/compiler/source_freshness/_helpers/state.py deleted file mode 100644 index c3e559a007..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/state.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Shared source freshness state normalization and hashing helpers.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import UTC, datetime, timedelta - -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.types import SourceFreshnessComparableRecord -from sqlbuild.cursor_algebra.constants import MINUTE_TO_DAY_DURATION_UNITS -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind - - -def normalize_source_freshness_data_version( - *, value: object, value_kind: SourceFreshnessValueKind -) -> str: - """Normalize a source freshness value for stable state storage.""" - - match value_kind: - case SourceFreshnessValueKind.TIMESTAMP: - if not isinstance(value, datetime): - raise SourceFreshnessObservationError( - "Timestamp source freshness values must be datetime values" - ) - normalized: datetime = value - if normalized.tzinfo is not None: - normalized = normalized.astimezone(UTC) - return normalized.isoformat() - case SourceFreshnessValueKind.INTEGER: - if isinstance(value, bool) or not isinstance(value, int): - raise SourceFreshnessObservationError( - "Integer source freshness values must be integer values" - ) - return str(value) - case SourceFreshnessValueKind.STRING: - if not isinstance(value, str): - raise SourceFreshnessObservationError( - "String source freshness values must be string values" - ) - return value - raise SourceFreshnessObservationError(f"Unsupported source freshness value kind: {value_kind}") - - -def source_freshness_data_version_hash( - *, - source_name: str, - strategy: SourceFreshnessStrategy | str, - value_kind: SourceFreshnessValueKind | str, - data_version: str, -) -> str: - """Hash the stable source freshness identity used as a graph input.""" - - payload: dict[str, str] = { - "source_name": source_name, - "strategy": strategy.value if isinstance(strategy, SourceFreshnessStrategy) else strategy, - "value_kind": value_kind.value - if isinstance(value_kind, SourceFreshnessValueKind) - else value_kind, - "data_version": data_version, - } - encoded: bytes = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() - return hashlib.sha256(encoded).hexdigest() - - -def source_freshness_records_equivalent( - *, - previous_record: SourceFreshnessComparableRecord, - current_record: SourceFreshnessComparableRecord, - lag_tolerance: str | None = None, -) -> bool: - """Return whether two source freshness records are equivalent for skip decisions.""" - - if previous_record.data_version_hash == current_record.data_version_hash: - return True - if previous_record.value_kind != SourceFreshnessValueKind.TIMESTAMP.value: - return False - if current_record.value_kind != SourceFreshnessValueKind.TIMESTAMP.value: - return False - if previous_record.data_version is None or current_record.data_version is None: - return False - previous_timestamp: datetime = _parse_timestamp_data_version(previous_record.data_version) - current_timestamp: datetime = _parse_timestamp_data_version(current_record.data_version) - if current_timestamp == previous_timestamp: - return True - if lag_tolerance is None: - return False - if current_timestamp < previous_timestamp: - return False - return current_timestamp - previous_timestamp <= _parse_lag_tolerance(lag_tolerance) - - -def _parse_timestamp_data_version(value: str) -> datetime: - try: - parsed: datetime = datetime.fromisoformat(value) - except ValueError as exc: - raise SourceFreshnessObservationError( - f"Timestamp source freshness state value is not valid ISO datetime: {value}" - ) from exc - if parsed.tzinfo is None or parsed.utcoffset() is None: - return parsed.replace(tzinfo=UTC) - return parsed.astimezone(UTC) - - -def _parse_lag_tolerance(value: str) -> timedelta: - duration: Duration | None = Duration.parse(value) - if duration is None or not duration.is_single_unit_in(MINUTE_TO_DAY_DURATION_UNITS): - raise SourceFreshnessObservationError( - "source freshness lag_tolerance must be a positive duration like 15m, 2h, or 1d" - ) - return timedelta(seconds=duration.fixed_seconds) diff --git a/src/sqlbuild/compiler/source_freshness/main/age_policy.py b/src/sqlbuild/compiler/source_freshness/main/age_policy.py deleted file mode 100644 index b353fbfe85..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/age_policy.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public source freshness age policy evaluation entrypoint.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild.compiler.source_freshness._helpers.age_policy import ( - evaluate_source_freshness_age_policy as _evaluate_source_freshness_age_policy, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus -from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy - - -def evaluate_source_freshness_age_policy( - *, - policy: SourceFreshnessAgePolicy | None, - data_version: object, - observed_at: datetime, -) -> SourceFreshnessAgeStatus | None: - """Evaluate optional timestamp age policy for one source freshness observation.""" - - return _evaluate_source_freshness_age_policy( - policy=policy, - data_version=data_version, - observed_at=observed_at, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/data_version_hash.py b/src/sqlbuild/compiler/source_freshness/main/data_version_hash.py deleted file mode 100644 index e291e8245d..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/data_version_hash.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public shared source freshness data-version hash entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.source_freshness._helpers.state import ( - source_freshness_data_version_hash as _source_freshness_data_version_hash, -) -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind - - -def source_freshness_data_version_hash( - *, - source_name: str, - strategy: SourceFreshnessStrategy | str, - value_kind: SourceFreshnessValueKind | str, - data_version: str, -) -> str: - """Hash the stable source freshness identity used as a graph input.""" - - return _source_freshness_data_version_hash( - source_name=source_name, - strategy=strategy, - value_kind=value_kind, - data_version=data_version, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/normalization.py b/src/sqlbuild/compiler/source_freshness/main/normalization.py deleted file mode 100644 index 338f95aeb5..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/normalization.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public shared source freshness normalization entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.source_freshness._helpers.state import ( - normalize_source_freshness_data_version as _normalize_source_freshness_data_version, -) -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind - - -def normalize_source_freshness_data_version( - *, value: object, value_kind: SourceFreshnessValueKind -) -> str: - """Normalize a source freshness value for stable state storage.""" - - return _normalize_source_freshness_data_version(value=value, value_kind=value_kind) diff --git a/src/sqlbuild/compiler/source_freshness/main/record_equivalence.py b/src/sqlbuild/compiler/source_freshness/main/record_equivalence.py deleted file mode 100644 index 35bbd2a6f1..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/record_equivalence.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public source freshness record equivalence entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.source_freshness._helpers.state import ( - source_freshness_records_equivalent as _source_freshness_records_equivalent, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessComparableRecord - - -def source_freshness_records_equivalent( - *, - previous_record: SourceFreshnessComparableRecord, - current_record: SourceFreshnessComparableRecord, - lag_tolerance: str | None = None, -) -> bool: - """Return whether two source freshness records are equivalent for skip decisions.""" - - return _source_freshness_records_equivalent( - previous_record=previous_record, - current_record=current_record, - lag_tolerance=lag_tolerance, - ) diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py index ab0bb19620..31231a63b4 100644 --- a/src/sqlbuild/executor/migrations/_helpers/columns.py +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -5,13 +5,11 @@ from datetime import UTC, datetime from typing import Any +from sqlbuild._native import deterministic_column_migration_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.deterministic_column_event_id import ( - deterministic_column_migration_event_id, -) from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.main.write_column_event import write_column_migration_event from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py index 685e755c6c..2ddafcdb9f 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py @@ -5,11 +5,9 @@ from datetime import datetime from typing import Any +from sqlbuild._native import deterministic_old_name_view_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.deterministic_old_name_view_event_id import ( - deterministic_old_name_view_event_id, -) from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( read_old_name_view_events, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index 9603e3bb2e..45656d83b5 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -8,6 +8,7 @@ from datetime import UTC, datetime, timedelta from typing import Any +from sqlbuild._native import deterministic_migration_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.main.grants_for_columns import grants_for_columns @@ -19,9 +20,6 @@ from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME -from sqlbuild.compiler.migrations.main.deterministic_event_id import ( - deterministic_migration_event_id, -) from sqlbuild.compiler.migrations.main.read_recorded_column_renames import ( read_recorded_column_renames, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py index e028be67bd..e3cd213f9c 100644 --- a/src/sqlbuild/executor/migrations/_helpers/promotion.py +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -5,12 +5,10 @@ from datetime import UTC, datetime from typing import Any +from sqlbuild._native import deterministic_migration_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.deterministic_event_id import ( - deterministic_migration_event_id, -) from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation, OldNameViewEvent diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index dcce6ff0dd..e33c45671b 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime from typing import Any +from sqlbuild._native import deterministic_old_name_view_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo @@ -19,9 +20,6 @@ MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, ) -from sqlbuild.compiler.migrations.main.deterministic_old_name_view_event_id import ( - deterministic_old_name_view_event_id, -) from sqlbuild.compiler.migrations.main.read_old_name_view_histories import ( read_old_name_view_histories, ) diff --git a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py index 070b623da8..f55f2a1e62 100644 --- a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py @@ -7,7 +7,12 @@ import pytest -from sqlbuild._native import read_latest_fingerprints, write_fingerprint +from sqlbuild._native import ( + normalize_source_freshness_data_version, + read_latest_fingerprints, + source_freshness_data_version_hash, + write_fingerprint, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -25,12 +30,6 @@ from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.bigquery._test_types import ( BigQueryBuildFlowTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py index 16a61aaa99..13f1b813ed 100644 --- a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py @@ -6,7 +6,12 @@ import pytest -from sqlbuild._native import read_latest_fingerprints, write_fingerprint +from sqlbuild._native import ( + normalize_source_freshness_data_version, + read_latest_fingerprints, + source_freshness_data_version_hash, + write_fingerprint, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -24,12 +29,6 @@ from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.databricks._test_types import ( DatabricksBuildFlowTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py index 26c87ee52e..03070bec2b 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py @@ -7,6 +7,10 @@ import pytest +from sqlbuild._native import ( + normalize_source_freshness_data_version, + source_freshness_data_version_hash, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -22,12 +26,6 @@ from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeBuildFlowTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py index 91dbce4d32..328765c511 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py @@ -2,14 +2,12 @@ from datetime import UTC, datetime +from sqlbuild._native import source_freshness_data_version_hash from sqlbuild.adapter.contract.models import ( QueryResult, TableFreshnessMetadata, TableFreshnessRequest, ) -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.spec.contracts.models import ( SourceEntry, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py index 1813408fa0..e7d28eb393 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -4,6 +4,9 @@ from dataclasses import dataclass +from sqlbuild.compiler.migrations.models import MigrationRelation +from sqlbuild.compiler.migrations.types import OldNameViewEventType + @dataclass(frozen=True) class NewestEventTestCase: @@ -60,3 +63,40 @@ class OldNameViewAbsentColumnsTestCase: absent_columns: tuple[str, ...] expected_statuses: tuple[str, ...] expected_archive_name: str | None + + +@dataclass(frozen=True) +class PinnedMoveEventIdTestCase: + description: str + run_id: str + target_name: str | None + origin: MigrationRelation + destination: MigrationRelation + expected_event_id: str + + +@dataclass(frozen=True) +class PinnedFreshnessHashTestCase: + description: str + source_name: str + strategy: str + value_kind: str + data_version: str + expected_hash: str + + +@dataclass(frozen=True) +class PinnedColumnEventIdTestCase: + description: str + relation: MigrationRelation + origin_column: str + destination_column: str + expected_event_id: str + + +@dataclass(frozen=True) +class PinnedOldNameEventIdTestCase: + description: str + event_type: OldNameViewEventType + migration_event_id: str + expected_event_id: str diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py index 8d6e7e1191..c796410eca 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py @@ -4,13 +4,9 @@ from datetime import UTC, datetime, timedelta -from sqlbuild.compiler.migrations.main.deterministic_column_event_id import ( +from sqlbuild._native import ( deterministic_column_migration_event_id, -) -from sqlbuild.compiler.migrations.main.deterministic_event_id import ( deterministic_migration_event_id, -) -from sqlbuild.compiler.migrations.main.deterministic_old_name_view_event_id import ( deterministic_old_name_view_event_id, ) from sqlbuild.compiler.migrations.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py new file mode 100644 index 0000000000..e873a56e4a --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py @@ -0,0 +1,137 @@ +"""Persisted state identities pinned to the values the Python implementation stored.""" + +import pytest + +from sqlbuild._native import ( + deterministic_column_migration_event_id, + deterministic_migration_event_id, + deterministic_old_name_view_event_id, + source_freshness_data_version_hash, +) +from sqlbuild.compiler.migrations.models import MigrationRelation +from sqlbuild.compiler.migrations.types import OldNameViewEventType +from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( + PinnedColumnEventIdTestCase, + PinnedFreshnessHashTestCase, + PinnedMoveEventIdTestCase, + PinnedOldNameEventIdTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PinnedMoveEventIdTestCase( + description="move with a database-qualified origin", + run_id="run-1", + target_name="prod", + origin=MigrationRelation(database="Analytics", schema="Staging", name="Orders"), + destination=MigrationRelation(database=None, schema="Marts", name="orders_v2"), + expected_event_id="d6b1671e12032fd2efd6b3cd386bb1a7c6ca0fb0b23b70ac7460b35b0cf3eec2", + ), + PinnedMoveEventIdTestCase( + description="move with non-ASCII names and no target", + run_id="run-\u00fc", + target_name=None, + origin=MigrationRelation(database=None, schema=None, name="Stra\u00dfe"), + destination=MigrationRelation(database="DB", schema="S", name="T"), + expected_event_id="b35d6666a637e3ceff072be25d1847d304d618808b5d2504d8d3a7a159394d89", + ), + ], + ids=lambda case: case.description, +) +def test_given_move_when_hashing_natively_then_matches_stored_python_event_id( + test_case: PinnedMoveEventIdTestCase, +) -> None: + event_id: str = deterministic_migration_event_id( + run_id=test_case.run_id, + target_name=test_case.target_name, + origin=test_case.origin, + destination=test_case.destination, + ) + + assert event_id == test_case.expected_event_id + + +@pytest.mark.parametrize( + "test_case", + [ + PinnedColumnEventIdTestCase( + description="column rename on a qualified relation", + relation=MigrationRelation(database="Analytics", schema="Marts", name="Orders"), + origin_column="Customer_ID", + destination_column="customer_key", + expected_event_id="3c7209754b53f98b38b7dbea44c6a9f019f47cfa79a847edceea535095da352c", + ), + ], + ids=lambda case: case.description, +) +def test_given_column_rename_when_hashing_natively_then_matches_stored_python_event_id( + test_case: PinnedColumnEventIdTestCase, +) -> None: + event_id: str = deterministic_column_migration_event_id( + run_id="run-1", + target_name="prod", + relation=test_case.relation, + origin_column=test_case.origin_column, + destination_column=test_case.destination_column, + ) + + assert event_id == test_case.expected_event_id + + +@pytest.mark.parametrize( + "test_case", + [ + PinnedOldNameEventIdTestCase( + description="old-name view creation", + event_type=OldNameViewEventType.VIEW_CREATED, + migration_event_id="abc123", + expected_event_id="505320604de24bfcf9ca65ba63d373978bb5a26dedf83ad5917d8c7ab70efa73", + ), + ], + ids=lambda case: case.description, +) +def test_given_old_name_step_when_hashing_natively_then_matches_stored_python_event_id( + test_case: PinnedOldNameEventIdTestCase, +) -> None: + event_id: str = deterministic_old_name_view_event_id( + event_type=test_case.event_type, migration_event_id=test_case.migration_event_id + ) + + assert event_id == test_case.expected_event_id + + +@pytest.mark.parametrize( + "test_case", + [ + PinnedFreshnessHashTestCase( + description="timestamp source freshness version", + source_name="orders", + strategy="column", + value_kind="timestamp", + data_version="2024-01-01T00:00:00+00:00", + expected_hash="795f41b1b4ffbca0b6bb0b5da1a4ab3ad7a280c28e27e5289bd3c1a2732c5756", + ), + PinnedFreshnessHashTestCase( + description="string source freshness version with escapes", + source_name="cli\u00ebnts", + strategy="sql", + value_kind="string", + data_version='v"1', + expected_hash="7f6e8f6a16da5d7653b75617515e4643961a227202c378b84cba4d18f39a7d12", + ), + ], + ids=lambda case: case.description, +) +def test_given_data_version_when_hashing_natively_then_matches_stored_python_hash( + test_case: PinnedFreshnessHashTestCase, +) -> None: + data_version_hash: str = source_freshness_data_version_hash( + source_name=test_case.source_name, + strategy=test_case.strategy, + value_kind=test_case.value_kind, + data_version=test_case.data_version, + ) + + assert data_version_hash == test_case.expected_hash diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py b/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py index be6fd6545f..cadbc02b3b 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild.compiler.source_freshness._helpers.age_policy import ( - evaluate_source_freshness_age_policy, -) +from sqlbuild._native import evaluate_source_freshness_age_policy from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy from tests.unit.src.sqlbuild.compiler.source_freshness._helpers._test_types import ( SourceFreshnessAgePolicyDurationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py index f69c76ceed..1eb327b456 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py @@ -5,12 +5,10 @@ import pytest +from sqlbuild._native import normalize_source_freshness_data_version from sqlbuild.adapter.contract.models import TableFreshnessMetadata from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) from sqlbuild.compiler.source_freshness.main.observation import ( observe_configured_source_freshness, ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index b17c55f509..8ca26d561c 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -5,6 +5,7 @@ from types import MappingProxyType from typing import Any +from sqlbuild._native import source_freshness_data_version_hash from sqlbuild.adapter.contract.models import ( RelationLookup, TableFreshnessMetadata, @@ -17,9 +18,6 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.source_freshness._helpers.sql import build_read_latest_sql from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py index aea4a773a2..639d906e1b 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild.compiler.source_freshness.main.age_policy import ( - evaluate_source_freshness_age_policy, -) +from sqlbuild._native import evaluate_source_freshness_age_policy from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py index 8ac57c0346..6d56c58452 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py @@ -6,15 +6,13 @@ import pytest +from sqlbuild._native import source_freshness_data_version_hash from sqlbuild.adapter.contract.models import TableFreshnessMetadata, TableFreshnessRequest from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.main._planning import ( build_direct_source_freshness_planning_result, ) -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py index 3927e117ba..793d2035ba 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py @@ -4,13 +4,9 @@ import pytest -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( +from sqlbuild._native import ( normalize_source_freshness_data_version, -) -from sqlbuild.compiler.source_freshness.main.record_equivalence import ( + source_freshness_data_version_hash, source_freshness_records_equivalent, ) from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py index b4b5e1776a..83fc645c18 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -8,12 +8,10 @@ import duckdb +from sqlbuild._native import deterministic_column_migration_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.main.deterministic_column_event_id import ( - deterministic_column_migration_event_id, -) from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, From 7fe4d6cda84f25c21217e6de79111ec0c03c1bf0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:31:23 +0100 Subject: [PATCH 04/37] fix: read native freshness helpers from the extension in the freshness command --- .../src/bindings/_helpers/freshness/freshness_observe.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs index 2657a8e5bc..c4d55866a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs @@ -162,7 +162,7 @@ fn record<'py>( observation.getattr("value_kind")?.getattr("value")?, )?; fields.set_item("data_version", &data_version)?; - fields.set_item(NATIVE, data_version_hash)?; + fields.set_item("data_version_hash", data_version_hash)?; fields.set_item("observed_at", observation.getattr("observed_at")?)?; py.import(FRESHNESS_MODELS)? .getattr("SourceFreshnessRecord")? From 8da940aebd37db6794f05f10fd857c9289239c29 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:34:55 +0100 Subject: [PATCH 05/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index d647cf1141..4de5c7383d 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 88829 +remaining_lines = 88471 [[stays]] path = "executor" From 1d108cf5fcb537999ff937c8696b43c37e385401 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:54:35 +0100 Subject: [PATCH 06/37] refactor: normalize dbt profiles and write dbt init config natively --- .../src/dbt_profile/_helpers/adapters.rs | 209 +++++++++++++ .../src/dbt_profile/_helpers/mod.rs | 2 + .../src/dbt_profile/_helpers/output.rs | 59 ++++ .../src/dbt_profile/constants.rs | 50 ++++ .../sqlbuild-config/src/dbt_profile/errors.rs | 5 + .../dbt_profile/main/dbt_init_project_toml.rs | 50 ++++ .../src/dbt_profile/main/mod.rs | 2 + .../dbt_profile/main/normalize_dbt_profile.rs | 32 ++ crates/sqlbuild-config/src/dbt_profile/mod.rs | 8 + .../sqlbuild-config/src/dbt_profile/models.rs | 61 ++++ .../sqlbuild-config/src/dbt_profile/tests.rs | 6 + .../src/dbt_profile/tests/helpers.rs | 26 ++ .../src/dbt_profile/tests/test_dbt_profile.rs | 119 ++++++++ .../src/dbt_profile/tests/test_types.rs | 14 + crates/sqlbuild-config/src/lib.rs | 1 + .../src/bindings/_helpers/dbt_interop/mod.rs | 2 + .../bindings/_helpers/dbt_interop/profile.rs | 160 ++++++++++ .../_helpers/dbt_interop/registration.rs | 9 + .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/main/register.rs | 5 +- src/sqlbuild/_native.pyi | 19 ++ .../dbt/_helpers/profile/connection.py | 2 +- .../integrations/dbt/_helpers/profile/init.py | 52 +--- .../dbt/_helpers/profile/normalize.py | 275 ------------------ .../dbt/profile/test_normalize.py | 2 +- 25 files changed, 842 insertions(+), 329 deletions(-) create mode 100644 crates/sqlbuild-config/src/dbt_profile/_helpers/adapters.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/_helpers/mod.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/_helpers/output.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/constants.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/errors.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/main/dbt_init_project_toml.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/main/mod.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/main/normalize_dbt_profile.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/mod.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/models.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/tests.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/tests/helpers.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_profile.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/profile/normalize.py diff --git a/crates/sqlbuild-config/src/dbt_profile/_helpers/adapters.rs b/crates/sqlbuild-config/src/dbt_profile/_helpers/adapters.rs new file mode 100644 index 0000000000..33f8cb933e --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/_helpers/adapters.rs @@ -0,0 +1,209 @@ +//! Per-adapter normalization of a rendered dbt profile output. + +use crate::dbt_profile::_helpers::output::Output; +use crate::dbt_profile::constants::{ + BIGQUERY_METHOD_ERROR_SUFFIX, BIGQUERY_SUPPORTED_METHODS, BIGQUERY_UNSUPPORTED_METHODS, + DATABRICKS_OAUTH_KEYS, DATABRICKS_PAT_AUTH_TYPE, DEFAULT_DATABRICKS_CATALOG, + DUCKDB_COPIED_KEYS, DUCKDB_UNSUPPORTED_KEYS, MEMORY_DATABASE, POSTGRES_COPIED_KEYS, + SNOWFLAKE_DROPPED_KEYS, SQLSERVER_SQL_AUTHENTICATION, +}; +use crate::dbt_profile::errors::DbtProfileError; +use crate::dbt_profile::models::{ConnectionKey, ConnectionSource, NormalizedProfile}; + +fn normalized( + adapter: &'static str, + connection: Vec<(ConnectionKey, ConnectionSource)>, +) -> NormalizedProfile { + NormalizedProfile { + adapter, + connection, + target_schema: None, + target_database: None, + warnings: Vec::new(), + } +} + +fn copied(output: &Output<'_>, keys: &[&'static str]) -> Vec<(ConnectionKey, ConnectionSource)> { + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = Vec::new(); + for key in keys { + if let Some(index) = output.index(key) { + connection.push((ConnectionKey::Name(key), ConnectionSource::Entry(index))); + } + } + connection +} + +fn renamed( + output: &Output<'_>, + (from, to): (&str, &'static str), +) -> Option<(ConnectionKey, ConnectionSource)> { + output + .index(from) + .map(|index| (ConnectionKey::Name(to), ConnectionSource::Entry(index))) +} + +pub(crate) fn duckdb(output: &Output<'_>) -> NormalizedProfile { + let database: ConnectionSource = output + .first_value(&["path"]) + .or_else(|| output.first_value(&["database"])) + .map_or( + ConnectionSource::Text(MEMORY_DATABASE), + ConnectionSource::Entry, + ); + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = + vec![(ConnectionKey::Name("database"), database)]; + connection.extend(copied(output, &DUCKDB_COPIED_KEYS)); + let unsupported: Vec = output.sorted_present(&DUCKDB_UNSUPPORTED_KEYS, false); + let mut profile: NormalizedProfile = normalized("duckdb", connection); + if !unsupported.is_empty() { + profile.warnings.push(format!( + "dbt-duckdb profile fields are not applied by SQLBuild yet: {}", + unsupported.join(", ") + )); + } + profile.target_schema = output.text_at(output.index("schema")); + profile.target_database = output.text_at(output.index("database")); + profile +} + +pub(crate) fn postgres(output: &Output<'_>) -> NormalizedProfile { + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = + copied(output, &POSTGRES_COPIED_KEYS); + if let Some(index) = output.first_value(&["dbname", "database"]) { + connection.push(( + ConnectionKey::Name("dbname"), + ConnectionSource::Entry(index), + )); + } + if let Some(index) = output.first_value(&["pass", "password"]) { + connection.push(( + ConnectionKey::Name("password"), + ConnectionSource::Entry(index), + )); + } + let mut profile: NormalizedProfile = normalized("postgres", connection); + profile.target_schema = output.non_empty_text(&["schema"]); + profile +} + +pub(crate) fn snowflake(output: &Output<'_>) -> NormalizedProfile { + let connection: Vec<(ConnectionKey, ConnectionSource)> = output + .0 + .iter() + .enumerate() + .filter(|(_, entry)| { + !entry + .key + .as_deref() + .is_some_and(|key| SNOWFLAKE_DROPPED_KEYS.contains(&key)) + }) + .map(|(index, _)| (ConnectionKey::Entry(index), ConnectionSource::Entry(index))) + .collect(); + let mut profile: NormalizedProfile = normalized("snowflake", connection); + profile.target_schema = output.non_empty_text(&["schema"]); + profile.target_database = output.non_empty_text(&["database"]); + profile +} + +fn bigquery_method_error(method: &str) -> DbtProfileError { + DbtProfileError(format!( + "dbt BigQuery profile method '{method}' {BIGQUERY_METHOD_ERROR_SUFFIX}" + )) +} + +pub(crate) fn bigquery(output: &Output<'_>) -> Result { + let method: Option = output.non_empty_text(&["method"]); + if let Some(method) = &method + && BIGQUERY_UNSUPPORTED_METHODS.contains(&method.as_str()) + { + return Err(bigquery_method_error(method)); + } + let project: Option = output.first_value(&["project", "database"]); + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = Vec::new(); + if let Some(index) = project { + connection.push(( + ConnectionKey::Name("project"), + ConnectionSource::Entry(index), + )); + } + connection.extend(renamed(output, ("location", "location"))); + connection.extend(renamed(output, ("keyfile", "credentials_path"))); + if let Some(method) = &method + && !BIGQUERY_SUPPORTED_METHODS.contains(&method.as_str()) + { + return Err(bigquery_method_error(method)); + } + let mut profile: NormalizedProfile = normalized("bigquery", connection); + profile.target_schema = output.non_empty_text(&["dataset", "schema"]); + profile.target_database = output.text_at(project).filter(|text| !text.is_empty()); + Ok(profile) +} + +pub(crate) fn databricks(output: &Output<'_>) -> Result { + let unsupported: Vec = output.sorted_present(&DATABRICKS_OAUTH_KEYS, true); + if let Some(auth_type) = output.non_empty_text(&["auth_type"]) + && auth_type != DATABRICKS_PAT_AUTH_TYPE + { + return Err(DbtProfileError(format!( + "dbt Databricks profile auth_type '{auth_type}' is not supported by SQLBuild dbt-profile normalization yet. Supported auth: personal access token" + ))); + } + if !unsupported.is_empty() { + return Err(DbtProfileError(format!( + "dbt Databricks OAuth/Azure profile fields are not supported by SQLBuild dbt-profile normalization yet: {}", + unsupported.join(", ") + ))); + } + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = Vec::new(); + connection.extend(renamed(output, ("host", "server_hostname"))); + connection.extend(renamed(output, ("http_path", "http_path"))); + connection.extend(renamed(output, ("token", "token"))); + let catalog: Option = output.first_value(&["catalog", "database"]); + connection.push(( + ConnectionKey::Name("catalog"), + catalog.map_or( + ConnectionSource::Text(DEFAULT_DATABRICKS_CATALOG), + ConnectionSource::Entry, + ), + )); + connection.extend(renamed(output, ("schema", "schema"))); + let mut profile: NormalizedProfile = normalized("databricks", connection); + profile.target_schema = output.non_empty_text(&["schema"]); + profile.target_database = match catalog { + Some(index) => output.text_at(Some(index)).filter(|text| !text.is_empty()), + None => Some(DEFAULT_DATABRICKS_CATALOG.to_owned()), + }; + Ok(profile) +} + +pub(crate) fn sqlserver(output: &Output<'_>) -> Result { + if output.is_true("windows_login") || output.is_true("trusted_connection") { + return Err(DbtProfileError( + "dbt SQL Server Windows authentication is not supported by SQLBuild dbt-profile normalization yet" + .to_owned(), + )); + } + if let Some(authentication) = output.non_empty_text(&["authentication", "auth"]) + && authentication.to_lowercase() != SQLSERVER_SQL_AUTHENTICATION + { + return Err(DbtProfileError(format!( + "dbt SQL Server authentication mode '{authentication}' is not supported by SQLBuild dbt-profile normalization yet. Supported authentication: sql" + ))); + } + let values: [(&'static str, Option); 5] = [ + ("host", output.first_value(&["server", "host"])), + ("port", output.first_value(&["port"])), + ("user", output.first_value(&["UID", "user", "username"])), + ("password", output.first_value(&["PWD", "password", "pass"])), + ("database", output.first_value(&["database"])), + ]; + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = Vec::new(); + for (key, index) in values { + if let Some(index) = index { + connection.push((ConnectionKey::Name(key), ConnectionSource::Entry(index))); + } + } + let mut profile: NormalizedProfile = normalized("sqlserver", connection); + profile.target_schema = output.non_empty_text(&["schema"]); + Ok(profile) +} diff --git a/crates/sqlbuild-config/src/dbt_profile/_helpers/mod.rs b/crates/sqlbuild-config/src/dbt_profile/_helpers/mod.rs new file mode 100644 index 0000000000..ded0f58ac0 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod adapters; +pub(crate) mod output; diff --git a/crates/sqlbuild-config/src/dbt_profile/_helpers/output.rs b/crates/sqlbuild-config/src/dbt_profile/_helpers/output.rs new file mode 100644 index 0000000000..4aadeb29ec --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/_helpers/output.rs @@ -0,0 +1,59 @@ +//! Lookups over a profile output with Python `dict` semantics. + +use crate::dbt_profile::models::{ProfileEntry, ProfileValue}; + +/// A profile output, read as Python reads the rendered `dict`. +pub(crate) struct Output<'a>(pub(crate) &'a [ProfileEntry]); + +impl Output<'_> { + /// The index of `key`, when present. + pub(crate) fn index(&self, key: &str) -> Option { + self.0 + .iter() + .position(|entry| entry.key.as_deref() == Some(key)) + } + + /// `output.get(keys[0], output.get(keys[1], ...))`: the first present key's index. + pub(crate) fn first_present(&self, keys: &[&str]) -> Option { + keys.iter().find_map(|key| self.index(key)) + } + + /// The first present key's index, unless its value is `None`. + pub(crate) fn first_value(&self, keys: &[&str]) -> Option { + self.first_present(keys) + .filter(|index| self.0[*index].value != ProfileValue::Null) + } + + /// `_string_or_none(output.get(...))`: a non-empty string value. + pub(crate) fn non_empty_text(&self, keys: &[&str]) -> Option { + self.text_at(self.first_present(keys)) + .filter(|text| !text.is_empty()) + } + + /// The string value at `index`, when it is a string. + pub(crate) fn text_at(&self, index: Option) -> Option { + match &self.0[index?].value { + ProfileValue::Text(text) => Some(text.clone()), + _ => None, + } + } + + /// Whether the value of `key` is exactly `True`. + pub(crate) fn is_true(&self, key: &str) -> bool { + self.index(key) + .is_some_and(|index| self.0[index].value == ProfileValue::Bool(true)) + } + + /// Named keys present in the output that are listed in `keys`, sorted. + pub(crate) fn sorted_present(&self, keys: &[&str], with_value: bool) -> Vec { + let mut present: Vec = self + .0 + .iter() + .filter(|entry| !with_value || entry.value != ProfileValue::Null) + .filter_map(|entry| entry.key.clone()) + .filter(|key| keys.contains(&key.as_str())) + .collect(); + present.sort_unstable(); + present + } +} diff --git a/crates/sqlbuild-config/src/dbt_profile/constants.rs b/crates/sqlbuild-config/src/dbt_profile/constants.rs new file mode 100644 index 0000000000..12b144478c --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/constants.rs @@ -0,0 +1,50 @@ +//! dbt profile output keys SQLBuild reads and the wording of unsupported profiles. + +pub(crate) const MEMORY_DATABASE: &str = ":memory:"; +pub(crate) const DEFAULT_DATABRICKS_CATALOG: &str = "hive_metastore"; +pub(crate) const DATABRICKS_PAT_AUTH_TYPE: &str = "pat"; +pub(crate) const SQLSERVER_SQL_AUTHENTICATION: &str = "sql"; +pub(crate) const BIGQUERY_SUPPORTED_METHODS: [&str; 2] = ["oauth", "service-account"]; +pub(crate) const BIGQUERY_UNSUPPORTED_METHODS: [&str; 3] = [ + "oauth-secrets", + "service-account-json", + "external-oauth-wif", +]; +pub(crate) const DUCKDB_COPIED_KEYS: [&str; 3] = ["extensions", "settings", "attach"]; +pub(crate) const DUCKDB_UNSUPPORTED_KEYS: [&str; 12] = [ + "config_options", + "disable_transactions", + "external_root", + "filesystems", + "is_ducklake", + "keep_open", + "module_paths", + "plugins", + "remote", + "retries", + "secrets", + "use_credential_provider", +]; +pub(crate) const POSTGRES_COPIED_KEYS: [&str; 11] = [ + "host", + "port", + "user", + "connect_timeout", + "role", + "keepalives_idle", + "sslmode", + "sslcert", + "sslkey", + "sslrootcert", + "application_name", +]; +pub(crate) const SNOWFLAKE_DROPPED_KEYS: [&str; 2] = ["type", "threads"]; +pub(crate) const DATABRICKS_OAUTH_KEYS: [&str; 6] = [ + "client_id", + "client_secret", + "azure_client_id", + "azure_client_secret", + "oauth_redirect_url", + "oauth_scopes", +]; +pub(crate) const BIGQUERY_METHOD_ERROR_SUFFIX: &str = "is not supported by SQLBuild dbt-profile normalization yet. Supported methods: oauth, service-account"; diff --git a/crates/sqlbuild-config/src/dbt_profile/errors.rs b/crates/sqlbuild-config/src/dbt_profile/errors.rs new file mode 100644 index 0000000000..fc3c81b94a --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/errors.rs @@ -0,0 +1,5 @@ +//! A dbt profile SQLBuild cannot connect with. + +/// `DbtProfileError(message)`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DbtProfileError(pub String); diff --git a/crates/sqlbuild-config/src/dbt_profile/main/dbt_init_project_toml.rs b/crates/sqlbuild-config/src/dbt_profile/main/dbt_init_project_toml.rs new file mode 100644 index 0000000000..f8ab22a2d1 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/main/dbt_init_project_toml.rs @@ -0,0 +1,50 @@ +//! The `sqlbuild_project.toml` that `sqb dbt init` writes. + +use sqlbuild_core::text::main::python_isalnum::python_isalnum; + +use crate::dbt_profile::models::DbtInitConfig; + +fn escaped(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") +} + +/// A bare TOML key when it is alphanumeric apart from `_` and `-`, else a quoted one. +fn table_key(value: &str) -> String { + let stripped: String = value.replace(['_', '-'], ""); + if !stripped.is_empty() && stripped.chars().all(python_isalnum) { + return value.to_owned(); + } + format!("\"{}\"", escaped(value)) +} + +/// The project config pointing SQLBuild at the dbt project and its profile target. +#[must_use] +pub fn dbt_init_project_toml(config: &DbtInitConfig) -> String { + let connection_name: String = format!("{}_dbt_profile", config.target_name); + let mut target_lines: Vec = vec![ + format!("[connections.{}]", table_key(&connection_name)), + "source = \"dbt_profile\"".to_owned(), + format!("profile = \"{}\"", escaped(&config.profile_name)), + format!("target = \"{}\"", escaped(&config.target_name)), + String::new(), + format!("[targets.{}]", table_key(&config.target_name)), + format!("connection = \"{}\"", escaped(&connection_name)), + ]; + if let Some(database) = &config.target_database { + target_lines.push(format!("database = \"{}\"", escaped(database))); + } + if let Some(schema) = &config.target_schema { + target_lines.push(format!("schema = \"{}\"", escaped(schema))); + } + format!( + "name = \"{}\"\nadapter = \"{}\"\ndefault_target = \"{}\"\n\n[dbt]\nproject_dir = \"{}\"\nprofiles_dir = \"{}\"\ntarget_path = \"{}\"\ntarget = \"{}\"\n\n{}\n", + escaped(&config.project_name), + escaped(&config.adapter), + escaped(&config.target_name), + escaped(&config.dbt_project_dir), + escaped(&config.profiles_dir), + escaped(&config.target_path), + escaped(&config.target_name), + target_lines.join("\n") + ) +} diff --git a/crates/sqlbuild-config/src/dbt_profile/main/mod.rs b/crates/sqlbuild-config/src/dbt_profile/main/mod.rs new file mode 100644 index 0000000000..d483631463 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/main/mod.rs @@ -0,0 +1,2 @@ +pub mod dbt_init_project_toml; +pub mod normalize_dbt_profile; diff --git a/crates/sqlbuild-config/src/dbt_profile/main/normalize_dbt_profile.rs b/crates/sqlbuild-config/src/dbt_profile/main/normalize_dbt_profile.rs new file mode 100644 index 0000000000..5772c1e9fe --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/main/normalize_dbt_profile.rs @@ -0,0 +1,32 @@ +//! Normalize a rendered dbt profile output into a SQLBuild connection. + +use crate::dbt_profile::_helpers::adapters::{ + bigquery, databricks, duckdb, postgres, snowflake, sqlserver, +}; +use crate::dbt_profile::_helpers::output::Output; +use crate::dbt_profile::errors::DbtProfileError; +use crate::dbt_profile::models::{NormalizedProfile, ProfileEntry}; + +/// The SQLBuild connection plan for a rendered profile output of type `adapter_type`. +pub fn normalize_dbt_profile( + entries: &[ProfileEntry], +) -> Result { + let output: Output<'_> = Output(entries); + let adapter_type: String = output.text_at(output.index("type")).unwrap_or_default(); + match adapter_type.as_str() { + "duckdb" => Ok(duckdb(&output)), + "postgres" => Ok(postgres(&output)), + "snowflake" => Ok(snowflake(&output)), + "bigquery" => bigquery(&output), + "databricks" => databricks(&output), + "sqlserver" => sqlserver(&output), + "motherduck" => Err(DbtProfileError( + "dbt profile type 'motherduck' is not supported directly. Use a dbt-duckdb profile with a MotherDuck path after SQLBuild MotherDuck profile support lands." + .to_owned(), + )), + _ => Err(DbtProfileError(format!( + "dbt profile type '{}' is not supported by SQLBuild dbt-profile connection resolution yet. Currently supported: bigquery, databricks, duckdb, postgres, snowflake, sqlserver", + if adapter_type.is_empty() { "" } else { adapter_type.as_str() } + ))), + } +} diff --git a/crates/sqlbuild-config/src/dbt_profile/mod.rs b/crates/sqlbuild-config/src/dbt_profile/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/dbt_profile/models.rs b/crates/sqlbuild-config/src/dbt_profile/models.rs new file mode 100644 index 0000000000..9f77770ac7 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/models.rs @@ -0,0 +1,61 @@ +//! A rendered dbt profile output and the SQLBuild connection built from it. + +/// One profile output value, as normalization tells values apart. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProfileValue { + /// `None`. + Null, + /// `True` or `False`. + Bool(bool), + Text(String), + /// Any other value; copied through unchanged. + Other, +} + +/// One `key: value` entry of a profile output, in output order; non-string keys have no name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProfileEntry { + pub key: Option, + pub value: ProfileValue, +} + +/// Where one connection value comes from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConnectionSource { + /// The value of the output entry at this index. + Entry(usize), + /// A literal default. + Text(&'static str), +} + +/// The key one connection value is written under. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConnectionKey { + Name(&'static str), + /// The key of the output entry at this index, unchanged. + Entry(usize), +} + +/// A normalized SQLBuild connection: its adapter, ordered values, target namespace and warnings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NormalizedProfile { + pub adapter: &'static str, + pub connection: Vec<(ConnectionKey, ConnectionSource)>, + pub target_schema: Option, + pub target_database: Option, + pub warnings: Vec, +} + +/// The values `sqb dbt init` writes into `sqlbuild_project.toml`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DbtInitConfig { + pub project_name: String, + pub adapter: String, + pub target_name: String, + pub profile_name: String, + pub dbt_project_dir: String, + pub profiles_dir: String, + pub target_path: String, + pub target_schema: Option, + pub target_database: Option, +} diff --git a/crates/sqlbuild-config/src/dbt_profile/tests.rs b/crates/sqlbuild-config/src/dbt_profile/tests.rs new file mode 100644 index 0000000000..003e0bb7d0 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dbt_profile.rs"] +mod dbt_profile; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/dbt_profile/tests/helpers.rs b/crates/sqlbuild-config/src/dbt_profile/tests/helpers.rs new file mode 100644 index 0000000000..ff983129d0 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/tests/helpers.rs @@ -0,0 +1,26 @@ +use crate::dbt_profile::models::{DbtInitConfig, ProfileEntry, ProfileValue}; + +pub(super) fn entry(key: &str, value: ProfileValue) -> ProfileEntry { + ProfileEntry { + key: Some(key.to_owned()), + value, + } +} + +pub(super) fn text(key: &str, value: &str) -> ProfileEntry { + entry(key, ProfileValue::Text(value.to_owned())) +} + +pub(super) fn init_config(target_name: &str, target_schema: Option<&str>) -> DbtInitConfig { + DbtInitConfig { + project_name: "analytics".to_owned(), + adapter: "duckdb".to_owned(), + target_name: target_name.to_owned(), + profile_name: "ana\"lytics".to_owned(), + dbt_project_dir: "../dbt_project".to_owned(), + profiles_dir: "C:\\profiles".to_owned(), + target_path: "../dbt_project/target".to_owned(), + target_schema: target_schema.map(str::to_owned), + target_database: None, + } +} diff --git a/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_profile.rs b/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_profile.rs new file mode 100644 index 0000000000..a16119e002 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_profile.rs @@ -0,0 +1,119 @@ +use crate::dbt_profile::errors::DbtProfileError; +use crate::dbt_profile::main::dbt_init_project_toml::dbt_init_project_toml; +use crate::dbt_profile::main::normalize_dbt_profile::normalize_dbt_profile; +use crate::dbt_profile::models::{ + ConnectionKey, ConnectionSource, NormalizedProfile, ProfileValue, +}; +use crate::dbt_profile::tests::helpers::{entry, init_config, text}; +use crate::dbt_profile::tests::test_types::{InitTomlTestCase, NormalizeTestCase}; + +#[test] +fn given_rendered_profile_outputs_when_normalizing_then_matches_python_rules() { + let test_cases = [ + NormalizeTestCase { + description: "duckdb falls back from a null path to memory and warns on plugins", + entries: vec![ + text("type", "duckdb"), + entry("path", ProfileValue::Null), + entry("plugins", ProfileValue::Other), + entry("extensions", ProfileValue::Other), + text("schema", ""), + ], + expected_result: Ok(NormalizedProfile { + adapter: "duckdb", + connection: vec![ + (ConnectionKey::Name("database"), ConnectionSource::Text(":memory:")), + (ConnectionKey::Name("extensions"), ConnectionSource::Entry(3)), + ], + target_schema: Some(String::new()), + target_database: None, + warnings: vec![ + "dbt-duckdb profile fields are not applied by SQLBuild yet: plugins".to_owned(), + ], + }), + }, + NormalizeTestCase { + description: "postgres prefers a present dbname even when it is null", + entries: vec![ + text("type", "postgres"), + entry("dbname", ProfileValue::Null), + text("database", "db"), + text("pass", "pw"), + text("host", "h"), + ], + expected_result: Ok(NormalizedProfile { + adapter: "postgres", + connection: vec![ + (ConnectionKey::Name("host"), ConnectionSource::Entry(4)), + (ConnectionKey::Name("password"), ConnectionSource::Entry(3)), + ], + target_schema: None, + target_database: None, + warnings: Vec::new(), + }), + }, + NormalizeTestCase { + description: "databricks defaults the catalog", + entries: vec![text("type", "databricks"), text("host", "h"), text("schema", "s")], + expected_result: Ok(NormalizedProfile { + adapter: "databricks", + connection: vec![ + (ConnectionKey::Name("server_hostname"), ConnectionSource::Entry(1)), + (ConnectionKey::Name("catalog"), ConnectionSource::Text("hive_metastore")), + (ConnectionKey::Name("schema"), ConnectionSource::Entry(2)), + ], + target_schema: Some("s".to_owned()), + target_database: Some("hive_metastore".to_owned()), + warnings: Vec::new(), + }), + }, + NormalizeTestCase { + description: "sqlserver rejects windows logins", + entries: vec![text("type", "sqlserver"), entry("windows_login", ProfileValue::Bool(true))], + expected_result: Err(DbtProfileError( + "dbt SQL Server Windows authentication is not supported by SQLBuild dbt-profile normalization yet".to_owned(), + )), + }, + NormalizeTestCase { + description: "missing type", + entries: vec![entry("type", ProfileValue::Other)], + expected_result: Err(DbtProfileError( + "dbt profile type '' is not supported by SQLBuild dbt-profile connection resolution yet. Currently supported: bigquery, databricks, duckdb, postgres, snowflake, sqlserver".to_owned(), + )), + }, + ]; + + for test_case in test_cases { + assert_eq!( + normalize_dbt_profile(&test_case.entries), + test_case.expected_result, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_init_values_when_writing_project_toml_then_keys_and_strings_are_escaped() { + let test_cases = [ + InitTomlTestCase { + description: "plain target with a schema", + config: init_config("dev", Some("main")), + expected_toml: "name = \"analytics\"\nadapter = \"duckdb\"\ndefault_target = \"dev\"\n\n[dbt]\nproject_dir = \"../dbt_project\"\nprofiles_dir = \"C:\\\\profiles\"\ntarget_path = \"../dbt_project/target\"\ntarget = \"dev\"\n\n[connections.dev_dbt_profile]\nsource = \"dbt_profile\"\nprofile = \"ana\\\"lytics\"\ntarget = \"dev\"\n\n[targets.dev]\nconnection = \"dev_dbt_profile\"\nschema = \"main\"\n", + }, + InitTomlTestCase { + description: "dotted target is quoted", + config: init_config("a.b", None), + expected_toml: "name = \"analytics\"\nadapter = \"duckdb\"\ndefault_target = \"a.b\"\n\n[dbt]\nproject_dir = \"../dbt_project\"\nprofiles_dir = \"C:\\\\profiles\"\ntarget_path = \"../dbt_project/target\"\ntarget = \"a.b\"\n\n[connections.\"a.b_dbt_profile\"]\nsource = \"dbt_profile\"\nprofile = \"ana\\\"lytics\"\ntarget = \"a.b\"\n\n[targets.\"a.b\"]\nconnection = \"a.b_dbt_profile\"\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + dbt_init_project_toml(&test_case.config), + test_case.expected_toml, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs b/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs new file mode 100644 index 0000000000..960caa3d8a --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::dbt_profile::errors::DbtProfileError; +use crate::dbt_profile::models::{DbtInitConfig, NormalizedProfile, ProfileEntry}; + +pub(super) struct NormalizeTestCase { + pub(super) description: &'static str, + pub(super) entries: Vec, + pub(super) expected_result: Result, +} + +pub(super) struct InitTomlTestCase { + pub(super) description: &'static str, + pub(super) config: DbtInitConfig, + pub(super) expected_toml: &'static str, +} diff --git a/crates/sqlbuild-config/src/lib.rs b/crates/sqlbuild-config/src/lib.rs index d2ffd151bf..504ad122f5 100644 --- a/crates/sqlbuild-config/src/lib.rs +++ b/crates/sqlbuild-config/src/lib.rs @@ -4,6 +4,7 @@ pub mod command_connection; pub(crate) mod constants; +pub mod dbt_profile; pub mod effective; pub mod errors; pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs new file mode 100644 index 0000000000..1583e51dd6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod profile; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs new file mode 100644 index 0000000000..678ca86dcd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs @@ -0,0 +1,160 @@ +//! dbt profile normalization and the project config `sqb dbt init` writes. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::dbt_profile::errors::DbtProfileError; +use sqlbuild_config::dbt_profile::main::dbt_init_project_toml::dbt_init_project_toml; +use sqlbuild_config::dbt_profile::main::normalize_dbt_profile::normalize_dbt_profile; +use sqlbuild_config::dbt_profile::models::{ + ConnectionKey, ConnectionSource, DbtInitConfig, NormalizedProfile, ProfileEntry, ProfileValue, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const DBT_ERRORS_MODULE: &str = "sqlbuild.integrations.dbt.exceptions"; +const INIT_TOML_OPTIONS: (&[&str], &[&str]) = ( + &[ + "project_name", + "adapter", + "target_name", + "profile_name", + "dbt_project_dir", + "profiles_dir", + "target_path", + "target_schema", + "target_database", + ], + &[], +); + +fn raised(py: Python<'_>, error: DbtProfileError) -> PyErr { + match imported(py, DBT_ERRORS_MODULE, "DbtProfileError") + .and_then(|class| class.call1((error.0,))) + { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +fn profile_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(ProfileValue::Null); + } + if value.is_instance_of::() { + return Ok(ProfileValue::Bool(value.is_truthy()?)); + } + if value.is_instance_of::() { + return Ok(ProfileValue::Text(value.extract()?)); + } + Ok(ProfileValue::Other) +} + +fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { + match value { + Some(text) => PyString::new(py, text).into_any(), + None => py.None().into_bound(py), + } +} + +/// `NormalizedDbtProfileConnection` with the connection built from `output`'s values. +fn normalized_connection<'py>( + output: &Bound<'py, PyDict>, + profile: &NormalizedProfile, +) -> PyResult> { + let py: Python<'py> = output.py(); + let items: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = output.iter().collect(); + let connection: Bound<'py, PyDict> = PyDict::new(py); + for (key, source) in &profile.connection { + let key: Bound<'py, PyAny> = match key { + ConnectionKey::Name(name) => PyString::new(py, name).into_any(), + ConnectionKey::Entry(index) => items[*index].0.clone(), + }; + let value: Bound<'py, PyAny> = match source { + ConnectionSource::Entry(index) => items[*index].1.clone(), + ConnectionSource::Text(text) => PyString::new(py, text).into_any(), + }; + connection.set_item(key, value)?; + } + call_with( + &imported(py, DBT_MODELS, "NormalizedDbtProfileConnection")?, + &[ + ("adapter", PyString::new(py, profile.adapter).into_any()), + ("connection", connection.into_any()), + ( + "target_schema", + optional_text(py, profile.target_schema.as_deref()), + ), + ( + "target_database", + optional_text(py, profile.target_database.as_deref()), + ), + ("warnings", PyTuple::new(py, &profile.warnings)?.into_any()), + ], + ) +} + +/// Normalize rendered dbt profile output for SQLBuild. +#[pyfunction] +#[pyo3(signature = (*, resolved))] +fn normalize_dbt_profile_connection<'py>( + py: Python<'py>, + resolved: Bound<'py, PyAny>, +) -> PyResult> { + let output: Bound<'py, PyDict> = PyDict::new(py); + output.update(&resolved.getattr("output")?.cast_into()?)?; + let mut entries: Vec = Vec::new(); + for (key, value) in output.iter() { + entries.push(ProfileEntry { + key: if key.is_instance_of::() { + Some(key.extract()?) + } else { + None + }, + value: profile_value(&value)?, + }); + } + let profile: NormalizedProfile = + normalize_dbt_profile(&entries).map_err(|error| raised(py, error))?; + normalized_connection(&output, &profile) +} + +fn text_argument(arguments: &KeywordArguments<'_>, name: &str) -> PyResult { + arguments.value(name)?.extract() +} + +/// The `sqlbuild_project.toml` text `sqb dbt init` writes. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_dbt_init_project_toml( + py: Python<'_>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "build_dbt_init_project_toml", + options, + INIT_TOML_OPTIONS, + )?; + Ok(dbt_init_project_toml(&DbtInitConfig { + project_name: text_argument(&arguments, "project_name")?, + adapter: text_argument(&arguments, "adapter")?, + target_name: text_argument(&arguments, "target_name")?, + profile_name: text_argument(&arguments, "profile_name")?, + dbt_project_dir: text_argument(&arguments, "dbt_project_dir")?, + profiles_dir: text_argument(&arguments, "profiles_dir")?, + target_path: text_argument(&arguments, "target_path")?, + target_schema: arguments.value("target_schema")?.extract()?, + target_database: arguments.value("target_database")?.extract()?, + })) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(normalize_dbt_profile_connection, module)?)?; + module.add_function(wrap_pyfunction!(build_dbt_init_project_toml, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs new file mode 100644 index 0000000000..667e3c8c19 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -0,0 +1,9 @@ +//! Register the static `sqb dbt` interop helpers. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::dbt_interop::profile; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + profile::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 8ef9476d7f..e5bc775799 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -17,6 +17,7 @@ pub(crate) mod contracts; pub(crate) mod cost_report; pub(crate) mod cursor; pub(crate) mod dag_artifact; +pub(crate) mod dbt_interop; pub(crate) mod debug_command; pub(crate) mod discovery; pub(crate) mod fingerprints; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 37ec76b7b2..36b87cafca 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -6,8 +6,8 @@ use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, clone, command_runtime, compile_attachment, compile_command, compile_driver, compile_outputs, - compiled_project, contracts, cost_report, cursor, dag_artifact, debug_command, discovery, - fingerprints, freshness, frontier, graph, janitor, lineage, macro_bridge, manifest, + compiled_project, contracts, cost_report, cursor, dag_artifact, dbt_interop, debug_command, + discovery, fingerprints, freshness, frontier, graph, janitor, lineage, macro_bridge, manifest, model_config, plan_output, presentation_text, project_assembly, refactor, references, render, result_output, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, workspace_commands, @@ -54,6 +54,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { result_output::registration::register(module)?; cost_report::registration::register(module)?; debug_command::registration::register(module)?; + dbt_interop::registration::register(module)?; command_runtime::registration::register(module)?; presentation_text::registration::register(module)?; workspace_commands::registration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 22c21490f5..a8a8ce8f52 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -226,6 +226,10 @@ from sqlbuild.executor.python_nodes.models import ( ) from sqlbuild.executor.scenario.models import ScenarioRunResult, ScenarioSnapshotCaptureRunResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult +from sqlbuild.integrations.dbt.models import ( + NormalizedDbtProfileConnection, + ResolvedDbtProfileOutput, +) from sqlbuild.lint.models import HeaderSpan, LintConfig, LintViolation from sqlbuild.lint.types import RelationKeyRow from sqlbuild.presentation.classes.cli_style import CliStyle @@ -1372,6 +1376,21 @@ def resolve_target_connection_config( def validate_preview_target( *, discovered_inputs: DiscoveredProjectInputs, as_target: str | None, command_name: str ) -> None: ... +def normalize_dbt_profile_connection( + *, resolved: ResolvedDbtProfileOutput +) -> NormalizedDbtProfileConnection: ... +def build_dbt_init_project_toml( + *, + project_name: str, + adapter: str, + target_name: str, + profile_name: str, + dbt_project_dir: str, + profiles_dir: str, + target_path: str, + target_schema: str | None, + target_database: str | None, +) -> str: ... def run_debug_command( *, project_dir: Path | None = None, diff --git a/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py b/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py index f2d81ace86..6f47f24aa8 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py +++ b/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py @@ -5,6 +5,7 @@ import sys from pathlib import Path +from sqlbuild._native import normalize_dbt_profile_connection from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.profile.load import ( @@ -13,7 +14,6 @@ load_raw_dbt_profile, select_dbt_profile_output, ) -from sqlbuild.integrations.dbt._helpers.profile.normalize import normalize_dbt_profile_connection from sqlbuild.integrations.dbt._helpers.profile.render import render_selected_dbt_profile_output from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import ( diff --git a/src/sqlbuild/integrations/dbt/_helpers/profile/init.py b/src/sqlbuild/integrations/dbt/_helpers/profile/init.py index de6fd49957..7eb32dcdf6 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/profile/init.py +++ b/src/sqlbuild/integrations/dbt/_helpers/profile/init.py @@ -6,13 +6,13 @@ import subprocess from pathlib import Path +from sqlbuild._native import build_dbt_init_project_toml, normalize_dbt_profile_connection from sqlbuild.integrations.dbt._helpers.profile.load import ( default_profiles_dir, load_dbt_project_metadata, load_raw_dbt_profile, select_dbt_profile_output, ) -from sqlbuild.integrations.dbt._helpers.profile.normalize import normalize_dbt_profile_connection from sqlbuild.integrations.dbt._helpers.profile.render import render_selected_dbt_profile_output from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import ( @@ -69,7 +69,7 @@ def build_dbt_init_project(*, request: DbtInitRequest) -> DbtInitResult: ) normalized: NormalizedDbtProfileConnection = normalize_dbt_profile_connection(resolved=resolved) target_path: Path = _resolve_path(root=dbt_project_dir, path=Path(metadata.target_path)) - toml: str = _build_project_toml( + toml: str = build_dbt_init_project_toml( project_name=metadata.project_name.replace("-", "_"), adapter=normalized.adapter, target_name=selected.target_name, @@ -178,54 +178,6 @@ def _display_path(*, path: Path, root: Path) -> str: return path.as_posix() -def _build_project_toml( - *, - project_name: str, - adapter: str, - target_name: str, - profile_name: str, - dbt_project_dir: str, - profiles_dir: str, - target_path: str, - target_schema: str | None, - target_database: str | None, -) -> str: - connection_name: str = f"{target_name}_dbt_profile" - target_lines: list[str] = [ - f"[connections.{_quote_key(connection_name)}]", - 'source = "dbt_profile"', - f'profile = "{_escape(profile_name)}"', - f'target = "{_escape(target_name)}"', - "", - f"[targets.{_quote_key(target_name)}]", - f'connection = "{_escape(connection_name)}"', - ] - if target_database is not None: - target_lines.append(f'database = "{_escape(target_database)}"') - if target_schema is not None: - target_lines.append(f'schema = "{_escape(target_schema)}"') - return ( - f'name = "{_escape(project_name)}"\n' - f'adapter = "{_escape(adapter)}"\n' - f'default_target = "{_escape(target_name)}"\n\n' - "[dbt]\n" - f'project_dir = "{_escape(dbt_project_dir)}"\n' - f'profiles_dir = "{_escape(profiles_dir)}"\n' - f'target_path = "{_escape(target_path)}"\n' - f'target = "{_escape(target_name)}"\n\n' + "\n".join(target_lines) + "\n" - ) - - -def _quote_key(value: str) -> str: - if value.replace("_", "").replace("-", "").isalnum(): - return value - return f'"{_escape(value)}"' - - -def _escape(value: str) -> str: - return value.replace("\\", "\\\\").replace('"', '\\"') - - def _dbt_debug_warnings( *, dbt_project_dir: Path, profiles_dir: Path, target_name: str ) -> tuple[str, ...]: diff --git a/src/sqlbuild/integrations/dbt/_helpers/profile/normalize.py b/src/sqlbuild/integrations/dbt/_helpers/profile/normalize.py deleted file mode 100644 index 24176361c4..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/profile/normalize.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Normalize rendered dbt profile outputs into SQLBuild connection config.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.models import ( - NormalizedDbtProfileConnection, - ResolvedDbtProfileOutput, -) - -_DBT_BIGQUERY_SUPPORTED_METHODS: frozenset[str] = frozenset({"oauth", "service-account"}) -_DBT_BIGQUERY_UNSUPPORTED_METHODS: frozenset[str] = frozenset( - {"oauth-secrets", "service-account-json", "external-oauth-wif"} -) -_DBT_DATABRICKS_PAT_AUTH_TYPE: str = "pat" -_DBT_DUCKDB_UNSUPPORTED_PROFILE_KEYS: frozenset[str] = frozenset( - { - "config_options", - "disable_transactions", - "external_root", - "filesystems", - "is_ducklake", - "keep_open", - "module_paths", - "plugins", - "remote", - "retries", - "secrets", - "use_credential_provider", - } -) -_DBT_PROFILE_HOST_KEY: str = "host" -_DBT_PROFILE_HTTP_PATH_KEY: str = "http_path" -_DBT_PROFILE_KEYFILE_KEY: str = "keyfile" -_DBT_PROFILE_LOCATION_KEY: str = "location" -_DBT_PROFILE_SCHEMA_KEY: str = "schema" -_DBT_PROFILE_TOKEN_KEY: str = "token" -_DBT_SQLSERVER_SQL_AUTHENTICATION: str = "sql" - - -def normalize_dbt_profile_connection( - *, resolved: ResolvedDbtProfileOutput -) -> NormalizedDbtProfileConnection: - """Normalize rendered dbt profile output for SQLBuild.""" - - adapter_type: str = resolved.adapter_type - try: - builtin_adapter: BuiltinAdapter = BuiltinAdapter(adapter_type) - except ValueError as exc: - raise DbtProfileError( - f"dbt profile type '{adapter_type or ''}' is not supported by SQLBuild " - "dbt-profile connection resolution yet. Currently supported: " - "bigquery, databricks, duckdb, postgres, snowflake, sqlserver" - ) from exc - match builtin_adapter: - case BuiltinAdapter.DUCKDB: - return _normalize_duckdb(resolved=resolved) - case BuiltinAdapter.POSTGRES: - return _normalize_postgres(resolved=resolved) - case BuiltinAdapter.SNOWFLAKE: - return _normalize_snowflake(resolved=resolved) - case BuiltinAdapter.BIGQUERY: - return _normalize_bigquery(resolved=resolved) - case BuiltinAdapter.DATABRICKS: - return _normalize_databricks(resolved=resolved) - case BuiltinAdapter.SQLSERVER: - return _normalize_sqlserver(resolved=resolved) - case BuiltinAdapter.MOTHERDUCK: - raise DbtProfileError( - "dbt profile type 'motherduck' is not supported directly. Use a dbt-duckdb " - "profile with a MotherDuck path after SQLBuild MotherDuck profile support lands." - ) - - -def _normalize_duckdb(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - connection: dict[str, object] = {} - path_value: object | None = output.get("path") - database_value: object | None = output.get("database") - database: object = path_value if path_value is not None else database_value - if database is None: - database = ":memory:" - connection["database"] = database - for key in ("extensions", "settings", "attach"): - if key in output: - connection[key] = output[key] - warnings: list[str] = [] - unsupported_keys: tuple[str, ...] = tuple( - sorted(key for key in output if key in _DBT_DUCKDB_UNSUPPORTED_PROFILE_KEYS) - ) - if unsupported_keys: - warnings.append( - "dbt-duckdb profile fields are not applied by SQLBuild yet: " - + ", ".join(unsupported_keys) - ) - schema_value: object | None = output.get("schema") - target_schema: str | None = schema_value if isinstance(schema_value, str) else None - database_name: str | None = database_value if isinstance(database_value, str) else None - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.DUCKDB.value, - connection=connection, - target_schema=target_schema, - target_database=database_name, - warnings=tuple(warnings), - ) - - -def _normalize_postgres(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - connection: dict[str, object] = _copy_present( - output=output, - keys=( - "host", - "port", - "user", - "connect_timeout", - "role", - "keepalives_idle", - "sslmode", - "sslcert", - "sslkey", - "sslrootcert", - "application_name", - ), - ) - database: object | None = output.get("dbname", output.get("database")) - password: object | None = output.get("pass", output.get("password")) - if database is not None: - connection["dbname"] = database - if password is not None: - connection["password"] = password - target_schema: str | None = _string_or_none(output.get("schema")) - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.POSTGRES.value, - connection=connection, - target_schema=target_schema, - target_database=None, - ) - - -def _normalize_snowflake(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - connection: dict[str, object] = dict(output) - connection.pop("type", None) - connection.pop("threads", None) - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.SNOWFLAKE.value, - connection=connection, - target_schema=_string_or_none(output.get("schema")), - target_database=_string_or_none(output.get("database")), - ) - - -def _normalize_bigquery(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - method: str | None = _string_or_none(output.get("method")) - if method in _DBT_BIGQUERY_UNSUPPORTED_METHODS: - raise DbtProfileError( - "dbt BigQuery profile method " - f"'{method}' is not supported by SQLBuild dbt-profile normalization yet. " - "Supported methods: oauth, service-account" - ) - project: object | None = output.get("project", output.get("database")) - connection: dict[str, object] = {} - if project is not None: - connection["project"] = project - if _DBT_PROFILE_LOCATION_KEY in output: - connection["location"] = output[_DBT_PROFILE_LOCATION_KEY] - if _DBT_PROFILE_KEYFILE_KEY in output: - connection["credentials_path"] = output[_DBT_PROFILE_KEYFILE_KEY] - if method is not None and method not in _DBT_BIGQUERY_SUPPORTED_METHODS: - raise DbtProfileError( - "dbt BigQuery profile method " - f"'{method}' is not supported by SQLBuild dbt-profile normalization yet. " - "Supported methods: oauth, service-account" - ) - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.BIGQUERY.value, - connection=connection, - target_schema=_string_or_none(output.get("dataset", output.get("schema"))), - target_database=_string_or_none(project), - ) - - -def _normalize_databricks(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - unsupported_auth_keys: tuple[str, ...] = tuple( - sorted( - key - for key in ( - "client_id", - "client_secret", - "azure_client_id", - "azure_client_secret", - "oauth_redirect_url", - "oauth_scopes", - ) - if output.get(key) is not None - ) - ) - auth_type: str | None = _string_or_none(output.get("auth_type")) - if auth_type is not None and auth_type != _DBT_DATABRICKS_PAT_AUTH_TYPE: - raise DbtProfileError( - "dbt Databricks profile auth_type " - f"'{auth_type}' is not supported by SQLBuild dbt-profile normalization yet. " - "Supported auth: personal access token" - ) - if unsupported_auth_keys: - raise DbtProfileError( - "dbt Databricks OAuth/Azure profile fields are not supported by SQLBuild " - "dbt-profile normalization yet: " + ", ".join(unsupported_auth_keys) - ) - connection: dict[str, object] = {} - if _DBT_PROFILE_HOST_KEY in output: - connection["server_hostname"] = output[_DBT_PROFILE_HOST_KEY] - if _DBT_PROFILE_HTTP_PATH_KEY in output: - connection["http_path"] = output[_DBT_PROFILE_HTTP_PATH_KEY] - if _DBT_PROFILE_TOKEN_KEY in output: - connection["token"] = output[_DBT_PROFILE_TOKEN_KEY] - catalog: object | None = output.get("catalog", output.get("database")) - if catalog is None: - catalog = "hive_metastore" - connection["catalog"] = catalog - if _DBT_PROFILE_SCHEMA_KEY in output: - connection["schema"] = output[_DBT_PROFILE_SCHEMA_KEY] - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.DATABRICKS.value, - connection=connection, - target_schema=_string_or_none(output.get("schema")), - target_database=_string_or_none(catalog), - ) - - -def _normalize_sqlserver(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - if output.get("windows_login") is True or output.get("trusted_connection") is True: - raise DbtProfileError( - "dbt SQL Server Windows authentication is not supported by SQLBuild " - "dbt-profile normalization yet" - ) - authentication: str | None = _string_or_none(output.get("authentication", output.get("auth"))) - if authentication is not None and authentication.lower() != _DBT_SQLSERVER_SQL_AUTHENTICATION: - raise DbtProfileError( - "dbt SQL Server authentication mode " - f"'{authentication}' is not supported by SQLBuild dbt-profile normalization yet. " - "Supported authentication: sql" - ) - connection: dict[str, object] = {} - host: object | None = output.get("server", output.get("host")) - user: object | None = output.get("UID", output.get("user", output.get("username"))) - password: object | None = output.get("PWD", output.get("password", output.get("pass"))) - for key, value in ( - ("host", host), - ("port", output.get("port")), - ("user", user), - ("password", password), - ("database", output.get("database")), - ): - if value is not None: - connection[key] = value - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.SQLSERVER.value, - connection=connection, - target_schema=_string_or_none(output.get("schema")), - target_database=None, - ) - - -def _copy_present(*, output: dict[str, object], keys: tuple[str, ...]) -> dict[str, object]: - return {key: output[key] for key in keys if key in output} - - -def _string_or_none(value: object | None) -> str | None: - return value if isinstance(value, str) and value else None diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py index f5c72ebb29..dc0ec6ecd7 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.integrations.dbt._helpers.profile.normalize import normalize_dbt_profile_connection +from sqlbuild._native import normalize_dbt_profile_connection from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import ( NormalizedDbtProfileConnection, From 9b863bf7c8526bd5cc9f1ef09de66d909bbffae7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:09:51 +0100 Subject: [PATCH 07/37] refactor: build and render the sqb dbt plan natively --- .../src/cli_text/_helpers/dbt_plan.rs | 197 ++++++++++++++ .../src/cli_text/_helpers/dbt_plan_json.rs | 92 +++++++ .../src/cli_text/_helpers/mod.rs | 2 + .../sqlbuild-core/src/cli_text/constants.rs | 3 + .../main/dbt_interop_plan_warnings.rs | 18 ++ .../cli_text/main/format_dbt_interop_plan.rs | 38 +++ .../main/format_dbt_interop_plan_json.rs | 17 ++ crates/sqlbuild-core/src/cli_text/main/mod.rs | 3 + crates/sqlbuild-core/src/cli_text/models.rs | 43 ++- .../src/cli_text/tests/test_cli_text.rs | 56 +++- .../src/cli_text/tests/test_types.rs | 7 + .../src/bindings/_helpers/dbt_interop/mod.rs | 1 + .../src/bindings/_helpers/dbt_interop/plan.rs | 254 ++++++++++++++++++ .../_helpers/dbt_interop/registration.rs | 3 +- src/sqlbuild/_native.pyi | 23 ++ src/sqlbuild/cli/commands/main/dbt/_dbt.py | 5 +- .../dbt/_helpers/pipeline/execution_phases.py | 7 +- .../dbt/_helpers/planning/orchestration.py | 2 +- .../dbt/_helpers/planning/plan.py | 249 ----------------- .../dbt/main/pipeline/render_plan.py | 28 -- .../planning/_render_interop_plan_json.py | 10 - .../planning/_render_interop_plan_text.py | 16 -- .../sqlbuild/cli/commands/main/dbt/helpers.py | 2 +- .../sqlbuild/integrations/dbt/test_execute.py | 2 +- .../sqlbuild/integrations/dbt/test_plan.py | 10 +- 25 files changed, 764 insertions(+), 324 deletions(-) create mode 100644 crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan.rs create mode 100644 crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan_json.rs create mode 100644 crates/sqlbuild-core/src/cli_text/main/dbt_interop_plan_warnings.rs create mode 100644 crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan.rs create mode 100644 crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan_json.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/planning/plan.py delete mode 100644 src/sqlbuild/integrations/dbt/main/pipeline/render_plan.py delete mode 100644 src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_json.py delete mode 100644 src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_text.py diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan.rs b/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan.rs new file mode 100644 index 0000000000..9a02df2c7e --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan.rs @@ -0,0 +1,197 @@ +//! Sections of the `sqb dbt` plan text. + +use std::collections::BTreeMap; + +use crate::cli_text::_helpers::layout::tree_connector; +use crate::cli_text::_helpers::summary_text::overflow_line; +use crate::cli_text::constants::DBT_RESOURCE_FALLBACK_TYPE; +use crate::cli_text::models::{CliStyle, DbtInteropPlanText, DbtLsNodeText, DisplayLimit}; +use crate::text::main::python_splitlines::python_splitlines; + +const STYLE: CliStyle = CliStyle { use_color: true }; + +/// `visible_entries`: the entries shown under the display cap. +fn visible(entries: &[T], display: &DisplayLimit) -> usize { + display + .max_entries + .map_or(entries.len(), |limit| limit.min(entries.len())) +} + +fn display_name(node: &DbtLsNodeText) -> String { + if !node.fqn.is_empty() { + return node.fqn.join("."); + } + node.name + .clone() + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| node.unique_id.clone()) +} + +fn node_lines(nodes: &[&DbtLsNodeText], display: &DisplayLimit) -> Vec { + let shown: usize = visible(nodes, display); + let mut lines: Vec = Vec::new(); + for (index, node) in nodes[..shown].iter().enumerate() { + let last: bool = index + 1 == shown && nodes.len() <= shown; + lines.push(format!( + " {} {}", + tree_connector(STYLE, last), + STYLE.object_name(&display_name(node)) + )); + } + let hidden: i64 = i64::try_from(nodes.len() - shown).unwrap_or(i64::MAX); + lines.extend(overflow_line(hidden, " ", &display.overflow_flag)); + lines +} + +/// The dbt section: counts, command, selected nodes by resource type and required ids. +pub(crate) fn dbt_section(plan: &DbtInteropPlanText, display: &DisplayLimit) -> Vec { + let required: &[String] = &plan.dbt_required_unique_ids; + let mut lines: Vec = vec![format!( + "{} {}", + STYLE.dbt_label("dbt"), + STYLE.dbt_section(&format!( + "({} selected, {} required)", + plan.dbt_selected_unique_ids.len(), + required.len() + )) + )]; + if plan.dbt_skip_reason.is_some() { + lines.push(STYLE.muted(" skipped: no dbt work selected")); + return lines; + } + lines.push(format!( + " {}: {}", + STYLE.label("command"), + STYLE.command(&plan.dbt_argv.join(" ")) + )); + let mut sorted: Vec<(String, String, &DbtLsNodeText)> = plan + .dbt_nodes + .iter() + .map(|node| { + ( + node.resource_type.clone().unwrap_or_default(), + display_name(node), + node, + ) + }) + .collect(); + sorted.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + let mut by_type: BTreeMap> = BTreeMap::new(); + for (_, _, node) in sorted { + let resource_type: String = node + .resource_type + .clone() + .filter(|kind| !kind.is_empty()) + .unwrap_or_else(|| DBT_RESOURCE_FALLBACK_TYPE.to_owned()); + by_type.entry(resource_type).or_default().push(node); + } + for (resource_type, nodes) in &by_type { + lines.push(String::new()); + lines.push(format!( + " {}", + STYLE.plan_section(&format!("{resource_type} ({})", nodes.len())) + )); + lines.extend(node_lines(nodes, display)); + } + if !required.is_empty() { + lines.push(String::new()); + lines.push(STYLE.dbt_label(&format!(" required ({})", required.len()))); + let shown: usize = visible(required, display); + for (index, unique_id) in required[..shown].iter().enumerate() { + lines.push(format!( + " {} {}", + tree_connector(STYLE, index + 1 == shown), + STYLE.object_name(unique_id) + )); + } + } + lines +} + +/// The SQLBuild section: the rendered SQLBuild plan, or the selected model names. +pub(crate) fn sqlbuild_section( + plan: &DbtInteropPlanText, + sqlbuild_plan: Option<&str>, + display: &DisplayLimit, +) -> Vec { + let mut lines: Vec = + vec![STYLE.plan_section(&format!("SQLBuild ({})", plan.sqlbuild_models.len()))]; + if plan.sqlbuild_skip_reason.is_some() { + lines.push(STYLE.muted(" skipped: no SQLBuild work selected")); + return lines; + } + if let Some(rendered) = sqlbuild_plan { + lines.extend(python_splitlines(rendered).into_iter().map(|line| { + if line.is_empty() { + String::new() + } else { + format!(" {line}") + } + })); + return lines; + } + let shown: usize = visible(&plan.sqlbuild_models, display); + lines.extend( + plan.sqlbuild_models[..shown] + .iter() + .map(|name| format!(" {}", STYLE.object_name(name))), + ); + lines +} + +/// Anchor terms, path translations and warnings, each after a blank line when present. +pub(crate) fn trailing_sections(plan: &DbtInteropPlanText, display: &DisplayLimit) -> Vec { + let mut lines: Vec = Vec::new(); + if !plan.anchors.is_empty() { + lines.push(String::new()); + lines.push(STYLE.label("dbt selector anchors")); + let shown: usize = visible(&plan.anchors, display); + for (term, unique_ids) in &plan.anchors[..shown] { + lines.push(format!( + " {}: {} dbt resources", + STYLE.command(term), + unique_ids.len() + )); + } + } + if !plan.path_translations.is_empty() { + lines.push(String::new()); + lines.push(STYLE.label("path translations")); + for (original, translated) in &plan.path_translations { + lines.push(format!( + " {} -> {}", + STYLE.command(original), + STYLE.command(translated) + )); + } + } + if !plan.warnings.is_empty() { + lines.push(String::new()); + lines.push(STYLE.warning("Warnings")); + for warning in &plan.warnings { + lines.push(STYLE.warning(&format!(" {warning}"))); + } + } + lines +} + +/// The text with every `ESC[...m` color sequence removed. +pub(crate) fn without_ansi(text: &str) -> String { + let mut result: String = String::with_capacity(text.len()); + let mut rest: &str = text; + while let Some(start) = rest.find("\x1b[") { + result.push_str(&rest[..start]); + let after: &str = &rest[start + 2..]; + let digits: usize = after + .find(|character: char| !(character.is_ascii_digit() || character == ';')) + .unwrap_or(after.len()); + if after[digits..].starts_with('m') { + rest = &after[digits + 1..]; + } else { + result.push_str("\x1b["); + rest = after; + } + } + result.push_str(rest); + result +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan_json.rs b/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan_json.rs new file mode 100644 index 0000000000..6ca1a9a0a2 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan_json.rs @@ -0,0 +1,92 @@ +//! The JSON payload of a `sqb dbt` plan. + +use crate::cli_text::models::DbtInteropPlanText; +use crate::json::models::JsonValue; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn texts(values: &[String]) -> JsonValue { + JsonValue::Array(values.iter().map(|value| text(value)).collect()) +} + +fn argvs(values: &[Vec]) -> JsonValue { + JsonValue::Array(values.iter().map(|argv| texts(argv)).collect()) +} + +fn object(fields: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + fields + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + ) +} + +fn skip_reason(reason: Option<&String>) -> JsonValue { + reason.map_or(JsonValue::Null, |reason| text(reason)) +} + +/// The plan's dbt and SQLBuild work as the JSON payload Python built. +pub(crate) fn dbt_plan_payload(plan: &DbtInteropPlanText) -> JsonValue { + object(vec![ + ("command", text(&plan.command)), + ( + "dbt", + object(vec![ + ("argv", texts(&plan.dbt_argv)), + ("supplemental_argvs", argvs(&plan.supplemental_dbt_argvs)), + ("selected_unique_ids", texts(&plan.dbt_selected_unique_ids)), + ("required_unique_ids", texts(&plan.dbt_required_unique_ids)), + ( + "required_selector_terms", + texts(&plan.dbt_required_selector_terms), + ), + ("skipped", JsonValue::Bool(plan.dbt_skip_reason.is_some())), + ("skip_reason", skip_reason(plan.dbt_skip_reason.as_ref())), + ]), + ), + ( + "sqlbuild", + object(vec![ + ("argvs", argvs(&plan.sqlbuild_argvs)), + ("selected_models", texts(&plan.sqlbuild_models)), + ( + "skipped", + JsonValue::Bool(plan.sqlbuild_skip_reason.is_some()), + ), + ( + "skip_reason", + skip_reason(plan.sqlbuild_skip_reason.as_ref()), + ), + ]), + ), + ( + "anchors", + JsonValue::Array( + plan.anchors + .iter() + .map(|(term, unique_ids)| { + object(vec![ + ("term", text(term)), + ("unique_ids", texts(unique_ids)), + ]) + }) + .collect(), + ), + ), + ( + "path_translations", + JsonValue::Array( + plan.path_translations + .iter() + .map(|(original, translated)| { + object(vec![("from", text(original)), ("to", text(translated))]) + }) + .collect(), + ), + ), + ("warnings", texts(&plan.warnings)), + ]) +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs index f450e37ec1..f7c744da86 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod bounded_sections; +pub(crate) mod dbt_plan; +pub(crate) mod dbt_plan_json; pub(crate) mod diagnostics; pub(crate) mod layout; pub(crate) mod plan_document; diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index eae5ddd0a4..6e575a762f 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -6,6 +6,7 @@ pub(crate) const RED: &str = "\x1b[38;5;167m"; pub(crate) const GREEN: &str = "\x1b[32m"; pub(crate) const YELLOW: &str = "\x1b[33m"; pub(crate) const BLUE: &str = "\x1b[34m"; +pub(crate) const ORANGE: &str = "\x1b[38;5;208m"; pub(crate) const RESET: &str = "\x1b[0m"; pub(crate) const SUCCESS_STATUS_LABELS: [&str; 3] = ["OK", "PASS", "SUCCESS"]; pub(crate) const WARNING_STATUS_LABELS: [&str; 2] = ["WARN", "WARNING"]; @@ -84,3 +85,5 @@ pub(crate) const FOOTER_ERROR_LABELS: [&str; 4] = ["FAIL", "ERROR", "SYNC_FAIL", pub(crate) const FOOTER_SKIP_LABELS: [&str; 1] = ["SKIP"]; pub(crate) const COUNT_HEADER_OPENING: &str = " ("; pub(crate) const COUNT_HEADER_CLOSING: &str = ")"; +pub(crate) const DBT_NO_MATCH_WARNING: &str = "No dbt or SQLBuild resources matched the selection."; +pub(crate) const DBT_RESOURCE_FALLBACK_TYPE: &str = "resource"; diff --git a/crates/sqlbuild-core/src/cli_text/main/dbt_interop_plan_warnings.rs b/crates/sqlbuild-core/src/cli_text/main/dbt_interop_plan_warnings.rs new file mode 100644 index 0000000000..a5cd9572e1 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/dbt_interop_plan_warnings.rs @@ -0,0 +1,18 @@ +//! The warnings a `sqb dbt` plan reports. + +use crate::cli_text::constants::DBT_NO_MATCH_WARNING; + +/// `warnings`, plus the no-match warning when neither dbt nor SQLBuild has work. +#[must_use] +pub fn dbt_interop_plan_warnings(warnings: Vec, has_work: bool) -> Vec { + if has_work + || warnings + .iter() + .any(|warning| warning == DBT_NO_MATCH_WARNING) + { + return warnings; + } + let mut warnings: Vec = warnings; + warnings.push(DBT_NO_MATCH_WARNING.to_owned()); + warnings +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan.rs b/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan.rs new file mode 100644 index 0000000000..818b39267a --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan.rs @@ -0,0 +1,38 @@ +//! The human `sqb dbt` plan. + +use crate::cli_text::_helpers::dbt_plan::{ + dbt_section, sqlbuild_section, trailing_sections, without_ansi, +}; +use crate::cli_text::_helpers::layout::surface_header; +use crate::cli_text::models::{CliStyle, DbtInteropPlanText, DisplayLimit}; + +/// Selected dbt and SQLBuild work, colored or plain; `sqlbuild_plan` is the rendered SQLBuild plan. +#[must_use] +pub fn format_dbt_interop_plan( + plan: &DbtInteropPlanText, + sqlbuild_plan: Option<&str>, + (display, use_color): (&DisplayLimit, bool), +) -> String { + let mut dbt_ids: Vec<&String> = plan + .dbt_selected_unique_ids + .iter() + .chain(&plan.dbt_required_unique_ids) + .collect(); + dbt_ids.sort_unstable(); + dbt_ids.dedup(); + let selected: usize = dbt_ids.len() + plan.sqlbuild_models.len(); + let mut lines: Vec = vec![ + surface_header( + CliStyle { use_color: true }, + "Plan ready", + &format!("{selected} selected resources"), + ), + String::new(), + ]; + lines.extend(dbt_section(plan, display)); + lines.push(String::new()); + lines.extend(sqlbuild_section(plan, sqlbuild_plan, display)); + lines.extend(trailing_sections(plan, display)); + let text: String = lines.join("\n"); + if use_color { text } else { without_ansi(&text) } +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan_json.rs b/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan_json.rs new file mode 100644 index 0000000000..12915a64b3 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan_json.rs @@ -0,0 +1,17 @@ +//! `sqb dbt plan --json`. + +use crate::cli_text::_helpers::dbt_plan_json::dbt_plan_payload; +use crate::cli_text::models::DbtInteropPlanText; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// `json.dumps(payload, indent=2)` of the plan's dbt and SQLBuild work. +#[must_use] +pub fn format_dbt_interop_plan_json(plan: &DbtInteropPlanText) -> String { + let payload: JsonValue = dbt_plan_payload(plan); + dumps( + &payload, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + ) + .unwrap_or_default() +} diff --git a/crates/sqlbuild-core/src/cli_text/main/mod.rs b/crates/sqlbuild-core/src/cli_text/main/mod.rs index f64b0962c9..284e6fec08 100644 --- a/crates/sqlbuild-core/src/cli_text/main/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/main/mod.rs @@ -1,7 +1,10 @@ +pub mod dbt_interop_plan_warnings; pub mod format_aligned_name_value; pub mod format_compile_text; pub mod format_completion_line; pub mod format_count_noun; +pub mod format_dbt_interop_plan; +pub mod format_dbt_interop_plan_json; pub mod format_overflow_line; pub mod format_phase_line; pub mod format_plan_completion; diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index ac0d0a03b0..4ff7b7d80e 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -1,7 +1,7 @@ //! Plain data of the human compile report, and the semantic styling it is rendered with. use crate::cli_text::constants::{ - BLUE, BOLD, DIM, ERROR_STATUS_LABELS, GREEN, RED, RESET, SKIPPED_STATUS_LABELS, + BLUE, BOLD, DIM, ERROR_STATUS_LABELS, GREEN, ORANGE, RED, RESET, SKIPPED_STATUS_LABELS, SUCCESS_STATUS_LABELS, WARNING_STATUS_LABELS, YELLOW, }; @@ -70,6 +70,16 @@ impl CliStyle { self.apply("", text) } + #[must_use] + pub fn dbt_label(self, text: &str) -> String { + self.apply(ORANGE, text) + } + + #[must_use] + pub fn dbt_section(self, text: &str) -> String { + self.apply(BOLD, text) + } + #[must_use] pub fn success(self, text: &str) -> String { self.apply(GREEN, text) @@ -506,3 +516,34 @@ impl CliDocument { rendered } } + +/// One node `dbt ls` selected. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtLsNodeText { + pub unique_id: String, + pub resource_type: Option, + pub name: Option, + pub fqn: Vec, +} + +/// Everything a `sqb dbt` plan shows, in text and JSON. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtInteropPlanText { + /// `DbtInteropCommand` value. + pub command: String, + pub dbt_argv: Vec, + pub supplemental_dbt_argvs: Vec>, + pub dbt_nodes: Vec, + pub dbt_selected_unique_ids: Vec, + pub dbt_required_unique_ids: Vec, + pub dbt_required_selector_terms: Vec, + /// `DbtInteropSkipReason` value, when dbt is skipped. + pub dbt_skip_reason: Option, + pub sqlbuild_argvs: Vec>, + pub sqlbuild_models: Vec, + pub sqlbuild_skip_reason: Option, + /// Each anchor term with the dbt unique ids it matched. + pub anchors: Vec<(String, Vec)>, + pub path_translations: Vec<(String, String)>, + pub warnings: Vec, +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index e1b88d9e36..ddac7563c2 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -1,15 +1,17 @@ use crate::cli_text::_helpers::diagnostics::diagnostics_text; use crate::cli_text::main::format_compile_text::format_compile_text; +use crate::cli_text::main::format_dbt_interop_plan::format_dbt_interop_plan; use crate::cli_text::main::format_plan_completion::format_plan_completion; use crate::cli_text::main::format_plan_text::format_plan_text; use crate::cli_text::main::semantic_coverage_notice::semantic_coverage_notice; use crate::cli_text::models::{ - CliStyle, CompileTextReport, DiagnosticText, LineageNote, ModelLine, ModelPlanText, - PlanTextReport, PlanWarningText, RelatedText, + CliStyle, CompileTextReport, DbtInteropPlanText, DbtLsNodeText, DiagnosticText, DisplayLimit, + LineageNote, ModelLine, ModelPlanText, PlanTextReport, PlanWarningText, RelatedText, }; use crate::cli_text::tests::helpers::{diagnostic, location, plan_report, table_model}; use crate::cli_text::tests::test_types::{ - CompileTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, PlanTextTestCase, + CompileTextTestCase, DbtPlanTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, + PlanTextTestCase, }; /// Expectations recorded from the retired Python formatter. @@ -182,3 +184,51 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { ); } } + +/// Expectations recorded from the retired Python formatter. +#[test] +fn given_dbt_interop_plan_when_formatting_then_matches_python_text() { + let node = |unique_id: &str, resource_type: Option<&str>| DbtLsNodeText { + unique_id: unique_id.to_owned(), + resource_type: resource_type.map(str::to_owned), + name: None, + fqn: Vec::new(), + }; + let test_cases = [DbtPlanTextTestCase { + description: "grouped nodes with overflow, required ids, anchors and warnings", + plan: DbtInteropPlanText { + command: "plan".to_owned(), + dbt_argv: vec!["dbt".to_owned(), "ls".to_owned()], + dbt_nodes: vec![ + node("model.a.b", Some("model")), + node("model.a.a", Some("model")), + node("x", None), + ], + dbt_selected_unique_ids: vec![ + "model.a.a".to_owned(), + "model.a.b".to_owned(), + "x".to_owned(), + ], + dbt_required_unique_ids: vec!["model.a.r".to_owned()], + sqlbuild_models: vec!["orders".to_owned()], + anchors: vec![("a+".to_owned(), vec!["u".to_owned()])], + warnings: vec!["careful".to_owned()], + ..DbtInteropPlanText::default() + }, + max_entries: Some(1), + expected_text: "Plan ready 5 selected resources\n\ndbt (3 selected, 1 required)\n command: dbt ls\n\n model (2)\n ├── model.a.a\n ... and 1 more (use --verbose to show all)\n\n resource (1)\n └── x\n\n required (1)\n └── model.a.r\n\nSQLBuild (1)\n orders\n\ndbt selector anchors\n a+: 1 dbt resources\n\nWarnings\n careful", + }]; + + for test_case in test_cases { + let display: DisplayLimit = DisplayLimit { + max_entries: test_case.max_entries, + overflow_flag: "--verbose".to_owned(), + }; + assert_eq!( + format_dbt_interop_plan(&test_case.plan, None, (&display, false)), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs index 09c3dd0dda..ed807492b8 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs @@ -25,3 +25,10 @@ pub(super) struct PlanTextTestCase { pub(super) expected_text: &'static str, pub(super) expected_completion: &'static str, } + +pub(super) struct DbtPlanTextTestCase { + pub(super) description: &'static str, + pub(super) plan: crate::cli_text::models::DbtInteropPlanText, + pub(super) max_entries: Option, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs index 1583e51dd6..01385ad05a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod plan; pub(crate) mod profile; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs new file mode 100644 index 0000000000..82c1cbc183 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs @@ -0,0 +1,254 @@ +//! Build and render the `sqb dbt` selection plan. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::main::dbt_interop_plan_warnings::dbt_interop_plan_warnings; +use sqlbuild_core::cli_text::main::format_dbt_interop_plan::format_dbt_interop_plan; +use sqlbuild_core::cli_text::main::format_dbt_interop_plan_json::format_dbt_interop_plan_json; +use sqlbuild_core::cli_text::models::{DbtInteropPlanText, DbtLsNodeText, DisplayLimit}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const DBT_TYPES: &str = "sqlbuild.integrations.dbt.types"; +const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; +const BUILD_OPTIONS: (&[&str], &[&str]) = ( + &[ + "command", + "dbt_command_argv", + "dbt_ls_nodes", + "sqlbuild_command_argvs", + "selection", + ], + &[ + "dbt_required_selector_terms", + "supplemental_dbt_command_argvs", + "warnings", + "sqlbuild_plan_output", + ], +); +const RENDER_OPTIONS: (&[&str], &[&str]) = + (&["plan", "json_output", "use_color"], &["display_options"]); + +fn strings(value: &Bound<'_, PyAny>) -> PyResult> { + value.try_iter()?.map(|item| item?.extract()).collect() +} + +fn argvs(value: &Bound<'_, PyAny>) -> PyResult>> { + value.try_iter()?.map(|argv| strings(&argv?)).collect() +} + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +fn reason_value(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.getattr("value")?.extract()?)) +} + +fn ls_node(node: &Bound<'_, PyAny>) -> PyResult { + Ok(DbtLsNodeText { + unique_id: node.getattr("unique_id")?.extract()?, + resource_type: optional_text(&node.getattr("resource_type")?)?, + name: optional_text(&node.getattr("name")?)?, + fqn: strings(&node.getattr("fqn")?)?, + }) +} + +fn plan_text(plan: &Bound<'_, PyAny>) -> PyResult { + let selection: Bound<'_, PyAny> = plan.getattr("selection")?; + let anchor_ids: Bound<'_, PyAny> = selection.getattr("dbt_anchor_unique_ids_by_term")?; + let empty: Bound<'_, PyTuple> = PyTuple::empty(plan.py()); + let mut anchors: Vec<(String, Vec)> = Vec::new(); + for term in selection.getattr("dbt_anchor_terms")?.try_iter()? { + let term: Bound<'_, PyAny> = term?; + let unique_ids: Vec = strings(&anchor_ids.call_method1("get", (&term, &empty))?)?; + anchors.push((term.extract()?, unique_ids)); + } + let mut path_translations: Vec<(String, String)> = Vec::new(); + for pair in selection.getattr("path_translations")?.try_iter()? { + path_translations.push(pair?.extract()?); + } + Ok(DbtInteropPlanText { + command: plan.getattr("command")?.getattr("value")?.extract()?, + dbt_argv: strings(&plan.getattr("dbt_command_argv")?)?, + supplemental_dbt_argvs: argvs(&plan.getattr("supplemental_dbt_command_argvs")?)?, + dbt_nodes: plan + .getattr("dbt_selected_nodes")? + .try_iter()? + .map(|node| ls_node(&node?)) + .collect::>()?, + dbt_selected_unique_ids: strings(&plan.getattr("dbt_selected_unique_ids")?)?, + dbt_required_unique_ids: strings(&selection.getattr("dbt_required_unique_ids")?)?, + dbt_required_selector_terms: strings(&plan.getattr("dbt_required_selector_terms")?)?, + dbt_skip_reason: reason_value(&plan.getattr("dbt_skip_reason")?)?, + sqlbuild_argvs: argvs(&plan.getattr("sqlbuild_command_argvs")?)?, + sqlbuild_models: strings(&selection.getattr("sqlbuild_model_names")?)?, + sqlbuild_skip_reason: reason_value(&plan.getattr("sqlbuild_skip_reason")?)?, + anchors, + path_translations, + warnings: strings(&plan.getattr("warnings")?)?, + }) +} + +fn tuple_of<'py>(value: Bound<'py, PyAny>) -> PyResult> { + Ok(PyTuple::new(value.py(), value.try_iter()?.collect::>>()?)?.into_any()) +} + +fn tuple_of_tuples<'py>(value: Bound<'py, PyAny>) -> PyResult> { + let items: Vec> = value + .try_iter()? + .map(|item| tuple_of(item?)) + .collect::>()?; + Ok(PyTuple::new(value.py(), items)?.into_any()) +} + +fn empty_or<'py>(value: Bound<'py, PyAny>) -> Bound<'py, PyAny> { + if value.is_none() { + return PyTuple::empty(value.py()).into_any(); + } + value +} + +fn skip_reason<'py>(py: Python<'py>, has_work: bool, member: &str) -> PyResult> { + if has_work { + return Ok(py.None().into_bound(py)); + } + imported(py, DBT_TYPES, "DbtInteropSkipReason")?.getattr(member) +} + +/// Build a display-ready selection plan without dbt model state. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_dbt_interop_plan<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "build_dbt_interop_plan", options, BUILD_OPTIONS)?; + let command: Bound<'py, PyAny> = + imported(py, DBT_TYPES, "DbtInteropCommand")?.call1((arguments.value("command")?,))?; + let nodes: Bound<'py, PyAny> = tuple_of(arguments.value("dbt_ls_nodes")?)?; + let mut selected: Vec = Vec::new(); + for node in nodes.try_iter()? { + selected.push(node?.getattr("unique_id")?.extract()?); + } + selected.sort(); + let selection: Bound<'py, PyAny> = arguments.value("selection")?; + let dbt_has_work: bool = + !selected.is_empty() || selection.getattr("dbt_required_unique_ids")?.is_truthy()?; + let sqlbuild_has_work: bool = selection.getattr("sqlbuild_model_names")?.is_truthy()?; + let warnings: Vec = dbt_interop_plan_warnings( + strings(&empty_or(arguments.value("warnings")?))?, + dbt_has_work || sqlbuild_has_work, + ); + call_with( + &imported(py, DBT_MODELS, "DbtInteropPlan")?, + &[ + ("command", command), + ( + "dbt_command_argv", + tuple_of(arguments.value("dbt_command_argv")?)?, + ), + ("dbt_selected_nodes", nodes), + ( + "dbt_selected_unique_ids", + PyTuple::new(py, selected)?.into_any(), + ), + ( + "sqlbuild_command_argvs", + tuple_of_tuples(arguments.value("sqlbuild_command_argvs")?)?, + ), + ("selection", selection), + ( + "sqlbuild_plan_output", + arguments.value("sqlbuild_plan_output")?, + ), + ( + "dbt_required_selector_terms", + tuple_of(empty_or(arguments.value("dbt_required_selector_terms")?))?, + ), + ( + "supplemental_dbt_command_argvs", + tuple_of_tuples(empty_or(arguments.value("supplemental_dbt_command_argvs")?))?, + ), + ( + "dbt_skip_reason", + skip_reason(py, dbt_has_work, "NO_DBT_WORK")?, + ), + ( + "sqlbuild_skip_reason", + skip_reason(py, sqlbuild_has_work, "NO_SQLBUILD_WORK")?, + ), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ], + ) +} + +fn display_limit(options: &Bound<'_, PyAny>) -> PyResult { + Ok(DisplayLimit { + max_entries: options.getattr("max_entries_per_section")?.extract()?, + overflow_flag: options.getattr("overflow_flag")?.extract()?, + }) +} + +/// Render a dbt interop plan in the requested CLI output format. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_dbt_interop_plan(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "render_dbt_interop_plan", options, RENDER_OPTIONS)?; + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let text: DbtInteropPlanText = plan_text(&plan)?; + if arguments.flag("json_output")? { + return Ok(format_dbt_interop_plan_json(&text)); + } + let mut display: Bound<'_, PyAny> = arguments.value("display_options")?; + if !display.is_truthy()? { + display = imported(py, PRESENTATION_MODELS, "DisplayOptions")?.call0()?; + } + let use_color: bool = arguments.flag("use_color")?; + let plan_output: Bound<'_, PyAny> = plan.getattr("sqlbuild_plan_output")?; + let sqlbuild_plan: Option = + if text.sqlbuild_skip_reason.is_some() || plan_output.is_none() { + None + } else { + Some( + call_with( + &imported(py, NATIVE_MODULE, "format_plan")?, + &[ + ("plan", plan_output), + ( + "use_color", + PyBool::new(py, use_color).to_owned().into_any(), + ), + ("display_options", display.clone()), + ], + )? + .extract()?, + ) + }; + Ok(format!( + "\n{}", + format_dbt_interop_plan( + &text, + sqlbuild_plan.as_deref(), + (&display_limit(&display)?, use_color) + ) + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_dbt_interop_plan, module)?)?; + module.add_function(wrap_pyfunction!(render_dbt_interop_plan, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index 667e3c8c19..f2265bf19e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -2,8 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dbt_interop::profile; +use crate::bindings::_helpers::dbt_interop::{plan, profile}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + plan::register(module)?; profile::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a8a8ce8f52..34c2e0fb18 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -227,9 +227,13 @@ from sqlbuild.executor.python_nodes.models import ( from sqlbuild.executor.scenario.models import ScenarioRunResult, ScenarioSnapshotCaptureRunResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.integrations.dbt.models import ( + DbtInteropPlan, + DbtInteropSelectionResult, + DbtLsNode, NormalizedDbtProfileConnection, ResolvedDbtProfileOutput, ) +from sqlbuild.integrations.dbt.types import DbtInteropCommand from sqlbuild.lint.models import HeaderSpan, LintConfig, LintViolation from sqlbuild.lint.types import RelationKeyRow from sqlbuild.presentation.classes.cli_style import CliStyle @@ -1391,6 +1395,25 @@ def build_dbt_init_project_toml( target_schema: str | None, target_database: str | None, ) -> str: ... +def build_dbt_interop_plan( + *, + command: DbtInteropCommand | str, + dbt_command_argv: Sequence[str], + dbt_ls_nodes: Sequence[DbtLsNode], + sqlbuild_command_argvs: Sequence[Sequence[str]], + selection: DbtInteropSelectionResult, + dbt_required_selector_terms: Sequence[str] = (), + supplemental_dbt_command_argvs: Sequence[Sequence[str]] = (), + warnings: Sequence[str] = (), + sqlbuild_plan_output: PlanOutput | None = None, +) -> DbtInteropPlan: ... +def render_dbt_interop_plan( + *, + plan: DbtInteropPlan, + json_output: bool, + use_color: bool, + display_options: DisplayOptions | None = None, +) -> str: ... def run_debug_command( *, project_dir: Path | None = None, diff --git a/src/sqlbuild/cli/commands/main/dbt/_dbt.py b/src/sqlbuild/cli/commands/main/dbt/_dbt.py index 6e8e39dd0c..705c0e75a6 100644 --- a/src/sqlbuild/cli/commands/main/dbt/_dbt.py +++ b/src/sqlbuild/cli/commands/main/dbt/_dbt.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import PlanningProgressReporter +from sqlbuild._native import PlanningProgressReporter, render_dbt_interop_plan from sqlbuild.cli.commands._helpers.dbt.auto_init import ( ensure_sqlbuild_project_for_dbt_command, ) @@ -19,9 +19,6 @@ from sqlbuild.integrations.dbt.main.cli.validate_execution_args import validate_dbt_execution_args from sqlbuild.integrations.dbt.main.pipeline.execute import execute_dbt_interop_from_project from sqlbuild.integrations.dbt.main.pipeline.plan import plan_dbt_interop_from_project -from sqlbuild.integrations.dbt.main.pipeline.render_plan import ( - render_dbt_interop_plan, -) from sqlbuild.integrations.dbt.models import DbtInteropExecutionRequest, DbtInteropPlan from sqlbuild.integrations.dbt.types import DbtInteropCommand from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py index 9173ead186..1df7b8fb1f 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py @@ -4,7 +4,11 @@ from dataclasses import replace -from sqlbuild._native import ConnectionProgressReporter, apply_command_warehouse +from sqlbuild._native import ( + ConnectionProgressReporter, + apply_command_warehouse, + render_dbt_interop_plan, +) from sqlbuild.cli.commands.main.dbt.dbt_sqlbuild_work import execute_dbt_sqlbuild_work from sqlbuild.cli.commands.main.execution.connection_progress import ( build_connection_progress_reporter, @@ -18,7 +22,6 @@ render_dbt_execution_summary_footer, ) from sqlbuild.integrations.dbt._helpers.pipeline.plan_output import build_sqlbuild_plan_output -from sqlbuild.integrations.dbt.main.pipeline.render_plan import render_dbt_interop_plan from sqlbuild.integrations.dbt.main.profile._resolve_connection_config import ( resolve_connection_config, ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py index 350a8f07e8..e9c9b55707 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py +++ b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py @@ -4,6 +4,7 @@ from collections.abc import Sequence +from sqlbuild._native import build_dbt_interop_plan from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion from sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors import ( @@ -13,7 +14,6 @@ from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( resolve_sqlbuild_model_dbt_refs, ) -from sqlbuild.integrations.dbt._helpers.planning.plan import build_dbt_interop_plan from sqlbuild.integrations.dbt._helpers.selection.core import resolve_dbt_interop_sqlbuild_selection from sqlbuild.integrations.dbt._helpers.selection.selector_terms import dbt_fqn_selector_term from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py b/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py deleted file mode 100644 index 3ab0487e19..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py +++ /dev/null @@ -1,249 +0,0 @@ -"""Plan construction and formatting for ordinary dbt interop commands.""" - -from __future__ import annotations - -import json -import re -from collections import defaultdict -from collections.abc import Sequence - -from sqlbuild._native import append_overflow_line, format_plan, visible_entries -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.integrations.dbt.models import DbtInteropPlan, DbtInteropSelectionResult, DbtLsNode -from sqlbuild.integrations.dbt.types import DbtInteropCommand, DbtInteropSkipReason -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.surface_header import format_surface_header -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.models import DisplayOptions - -_ANSI_ESCAPE_PATTERN: re.Pattern[str] = re.compile(r"\033\[[0-9;]*m") - - -def build_dbt_interop_plan( - *, - command: DbtInteropCommand | str, - dbt_command_argv: Sequence[str], - dbt_ls_nodes: Sequence[DbtLsNode], - sqlbuild_command_argvs: Sequence[Sequence[str]], - selection: DbtInteropSelectionResult, - dbt_required_selector_terms: Sequence[str] = (), - supplemental_dbt_command_argvs: Sequence[Sequence[str]] = (), - warnings: Sequence[str] = (), - sqlbuild_plan_output: PlanOutput | None = None, -) -> DbtInteropPlan: - """Build a display-ready selection plan without dbt model state.""" - - normalized_command: DbtInteropCommand = DbtInteropCommand(command) - dbt_selected_unique_ids: tuple[str, ...] = tuple( - sorted(node.unique_id for node in dbt_ls_nodes) - ) - dbt_has_work: bool = bool(dbt_selected_unique_ids or selection.dbt_required_unique_ids) - sqlbuild_has_work: bool = bool(selection.sqlbuild_model_names) - resolved_warnings: tuple[str, ...] = tuple(warnings) - no_match_warning: str = "No dbt or SQLBuild resources matched the selection." - if not dbt_has_work and not sqlbuild_has_work and no_match_warning not in resolved_warnings: - resolved_warnings = (*resolved_warnings, no_match_warning) - return DbtInteropPlan( - command=normalized_command, - dbt_command_argv=tuple(dbt_command_argv), - dbt_selected_nodes=tuple(dbt_ls_nodes), - dbt_selected_unique_ids=dbt_selected_unique_ids, - sqlbuild_command_argvs=tuple(tuple(argv) for argv in sqlbuild_command_argvs), - selection=selection, - sqlbuild_plan_output=sqlbuild_plan_output, - dbt_required_selector_terms=tuple(dbt_required_selector_terms), - supplemental_dbt_command_argvs=tuple( - tuple(argv) for argv in supplemental_dbt_command_argvs - ), - dbt_skip_reason=None if dbt_has_work else DbtInteropSkipReason.NO_DBT_WORK, - sqlbuild_skip_reason=None if sqlbuild_has_work else DbtInteropSkipReason.NO_SQLBUILD_WORK, - warnings=resolved_warnings, - ) - - -def format_dbt_interop_plan( - *, - plan: DbtInteropPlan, - use_color: bool = True, - display_options: DisplayOptions | None = None, -) -> str: - """Format selected dbt and SQLBuild work without change classifications.""" - - options: DisplayOptions = display_options or DisplayOptions() - selected_count: int = len( - frozenset((*plan.dbt_selected_unique_ids, *plan.selection.dbt_required_unique_ids)) - ) + len(plan.selection.sqlbuild_model_names) - style: CliStyle = CliStyle(use_color=True) - lines: list[str] = [ - format_surface_header( - style=style, title="Plan ready", context=f"{selected_count} selected resources" - ), - "", - ] - lines = _format_dbt_section(lines=lines, plan=plan, options=options) - lines.append("") - lines = _format_sqlbuild_section(lines=lines, plan=plan, options=options, use_color=use_color) - lines = _format_anchor_section(lines=lines, plan=plan, options=options) - lines = _format_path_translation_section(lines=lines, plan=plan) - lines = _format_warning_section(lines=lines, plan=plan) - result: str = "\n".join(lines) - return result if use_color else _ANSI_ESCAPE_PATTERN.sub("", result) - - -def format_dbt_interop_plan_json(plan: DbtInteropPlan) -> str: - """Serialize an ordinary interop selection plan to stable JSON.""" - - result: dict[str, object] = { - "command": plan.command.value, - "dbt": { - "argv": list(plan.dbt_command_argv), - "supplemental_argvs": [list(argv) for argv in plan.supplemental_dbt_command_argvs], - "selected_unique_ids": list(plan.dbt_selected_unique_ids), - "required_unique_ids": list(plan.selection.dbt_required_unique_ids), - "required_selector_terms": list(plan.dbt_required_selector_terms), - "skipped": plan.dbt_skip_reason is not None, - "skip_reason": plan.dbt_skip_reason.value if plan.dbt_skip_reason is not None else None, - }, - "sqlbuild": { - "argvs": [list(argv) for argv in plan.sqlbuild_command_argvs], - "selected_models": list(plan.selection.sqlbuild_model_names), - "skipped": plan.sqlbuild_skip_reason is not None, - "skip_reason": ( - plan.sqlbuild_skip_reason.value if plan.sqlbuild_skip_reason is not None else None - ), - }, - "anchors": [ - { - "term": term, - "unique_ids": list(plan.selection.dbt_anchor_unique_ids_by_term.get(term, ())), - } - for term in plan.selection.dbt_anchor_terms - ], - "path_translations": [ - {"from": original, "to": translated} - for original, translated in plan.selection.path_translations - ], - "warnings": list(plan.warnings), - } - return json.dumps(result, indent=2) - - -def _format_dbt_section( - *, lines: list[str], plan: DbtInteropPlan, options: DisplayOptions -) -> list[str]: - style: CliStyle = CliStyle(use_color=True) - selected_count: int = len(plan.dbt_selected_unique_ids) - required_count: int = len(plan.selection.dbt_required_unique_ids) - lines.append( - f"{style.dbt_label('dbt')} " - f"{style.dbt_section(f'({selected_count} selected, {required_count} required)')}" - ) - if plan.dbt_skip_reason is not None: - lines.append(style.muted(" skipped: no dbt work selected")) - return lines - lines.append(f" {style.label('command')}: {style.command(' '.join(plan.dbt_command_argv))}") - nodes_by_type: dict[str, list[DbtLsNode]] = defaultdict(list) - for node in sorted(plan.dbt_selected_nodes, key=_dbt_node_sort_key): - nodes_by_type[node.resource_type or "resource"].append(node) - for resource_type, nodes in sorted(nodes_by_type.items()): - lines.append("") - lines.append(f" {style.plan_section(f'{resource_type} ({len(nodes)})')}") - visible_nodes: Sequence[DbtLsNode] = visible_entries(entries=nodes, options=options) - node_index: int - for node_index, node in enumerate(visible_nodes): - connector: str = tree_connector( - style=style, - last=node_index == len(visible_nodes) - 1 and len(nodes) <= len(visible_nodes), - ) - lines.append(f" {connector} {style.dbt_object_name(_dbt_node_display_name(node))}") - lines = append_overflow_line( - lines=lines, - total_count=len(nodes), - visible_count=len(visible_nodes), - indent=" ", - options=options, - ) - if required_count: - lines.append("") - lines.append(style.dbt_label(f" required ({required_count})")) - required_ids: Sequence[str] = visible_entries( - entries=plan.selection.dbt_required_unique_ids, - options=options, - ) - unique_id_index: int - for unique_id_index, unique_id in enumerate(required_ids): - connector = tree_connector(style=style, last=unique_id_index == len(required_ids) - 1) - lines.append(f" {connector} {style.dbt_object_name(unique_id)}") - return lines - - -def _format_sqlbuild_section( - *, - lines: list[str], - plan: DbtInteropPlan, - options: DisplayOptions, - use_color: bool, -) -> list[str]: - style: CliStyle = CliStyle(use_color=True) - lines.append(style.plan_section(f"SQLBuild ({len(plan.selection.sqlbuild_model_names)})")) - if plan.sqlbuild_skip_reason is not None: - lines.append(style.muted(" skipped: no SQLBuild work selected")) - return lines - if plan.sqlbuild_plan_output is not None: - rendered: str = format_plan( - plan=plan.sqlbuild_plan_output, - use_color=use_color, - display_options=options, - ) - lines.extend(f" {line}" if line else "" for line in rendered.splitlines()) - return lines - for name in visible_entries(entries=plan.selection.sqlbuild_model_names, options=options): - lines.append(f" {style.object_name(name)}") - return lines - - -def _format_anchor_section( - *, lines: list[str], plan: DbtInteropPlan, options: DisplayOptions -) -> list[str]: - if not plan.selection.dbt_anchor_terms: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.extend(("", style.label("dbt selector anchors"))) - visible_terms: Sequence[str] = visible_entries( - entries=plan.selection.dbt_anchor_terms, - options=options, - ) - for term in visible_terms: - count: int = len(plan.selection.dbt_anchor_unique_ids_by_term.get(term, ())) - lines.append(f" {style.command(term)}: {count} dbt resources") - return lines - - -def _format_path_translation_section(*, lines: list[str], plan: DbtInteropPlan) -> list[str]: - if not plan.selection.path_translations: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.extend(("", style.label("path translations"))) - for original, translated in plan.selection.path_translations: - lines.append(f" {style.command(original)} -> {style.command(translated)}") - return lines - - -def _format_warning_section(*, lines: list[str], plan: DbtInteropPlan) -> list[str]: - if not plan.warnings: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.extend(("", style.warning("Warnings"))) - for warning in plan.warnings: - lines.append(style.warning(f" {warning}")) - return lines - - -def _dbt_node_display_name(node: DbtLsNode) -> str: - if node.fqn: - return ".".join(node.fqn) - return node.name or node.unique_id - - -def _dbt_node_sort_key(node: DbtLsNode) -> tuple[str, str]: - return node.resource_type or "", _dbt_node_display_name(node) diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/render_plan.py b/src/sqlbuild/integrations/dbt/main/pipeline/render_plan.py deleted file mode 100644 index 2023e70b38..0000000000 --- a/src/sqlbuild/integrations/dbt/main/pipeline/render_plan.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Render dbt interop plan output for CLI callers.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt.main.planning._render_interop_plan_json import ( - format_dbt_interop_plan_json, -) -from sqlbuild.integrations.dbt.main.planning._render_interop_plan_text import ( - format_dbt_interop_plan, -) -from sqlbuild.integrations.dbt.models import DbtInteropPlan -from sqlbuild.presentation.models import DisplayOptions - - -def render_dbt_interop_plan( - *, - plan: DbtInteropPlan, - json_output: bool, - use_color: bool, - display_options: DisplayOptions | None = None, -) -> str: - """Render a dbt interop plan in the requested CLI output format.""" - - if json_output: - return format_dbt_interop_plan_json(plan) - return "\n" + format_dbt_interop_plan( - plan=plan, use_color=use_color, display_options=display_options - ) diff --git a/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_json.py b/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_json.py deleted file mode 100644 index 10768e319a..0000000000 --- a/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_json.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Format a dbt interop plan as JSON.""" - -from sqlbuild.integrations.dbt._helpers.planning.plan import format_dbt_interop_plan_json as _format -from sqlbuild.integrations.dbt.models import DbtInteropPlan - - -def format_dbt_interop_plan_json(plan: DbtInteropPlan) -> str: - """Serialize a dbt interop plan to stable JSON.""" - - return _format(plan) diff --git a/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_text.py b/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_text.py deleted file mode 100644 index 99cb325cc9..0000000000 --- a/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_text.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Format a dbt interop plan.""" - -from sqlbuild.integrations.dbt._helpers.planning.plan import format_dbt_interop_plan as _format -from sqlbuild.integrations.dbt.models import DbtInteropPlan -from sqlbuild.presentation.models import DisplayOptions - - -def format_dbt_interop_plan( - *, - plan: DbtInteropPlan, - use_color: bool = True, - display_options: DisplayOptions | None = None, -) -> str: - """Format a dbt interop plan for human output.""" - - return _format(plan=plan, use_color=use_color, display_options=display_options) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py index 9da1136f09..760b9d84af 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py @@ -2,7 +2,7 @@ from pathlib import Path -from sqlbuild.integrations.dbt._helpers.planning.plan import build_dbt_interop_plan +from sqlbuild._native import build_dbt_interop_plan from sqlbuild.integrations.dbt.models import DbtInteropPlan, DbtInteropSelectionResult from sqlbuild.integrations.dbt.types import DbtInteropCommand from tests.unit.src.sqlbuild.cli.commands.main.dbt._test_types import DbtAutoInitTestCase diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index 4dc3665ed0..aef13370a8 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import build_dbt_interop_plan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -16,7 +17,6 @@ build_failed_sqlbuild_model_names, build_merged_dbt_execution_argv, ) -from sqlbuild.integrations.dbt._helpers.planning.plan import build_dbt_interop_plan from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.main.pipeline import execute as execute_module from sqlbuild.integrations.dbt.main.pipeline import plan as plan_module diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py index 300370e5fa..f386c5a12c 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py @@ -2,11 +2,7 @@ import pytest -from sqlbuild.integrations.dbt._helpers.planning.plan import ( - build_dbt_interop_plan, - format_dbt_interop_plan, - format_dbt_interop_plan_json, -) +from sqlbuild._native import build_dbt_interop_plan, render_dbt_interop_plan from sqlbuild.integrations.dbt.models import DbtInteropPlan, DbtInteropSelectionResult, DbtLsNode from tests.unit.src.sqlbuild.integrations.dbt._test_types import DbtPlanTestCase @@ -86,8 +82,8 @@ def test_given_ordinary_interop_selection_when_rendering_then_reports_selected_w warnings=test_case.warnings, ) - human: str = format_dbt_interop_plan(plan=plan, use_color=False) - json_output: str = format_dbt_interop_plan_json(plan) + human: str = render_dbt_interop_plan(plan=plan, json_output=False, use_color=False) + json_output: str = render_dbt_interop_plan(plan=plan, json_output=True, use_color=False) assert (plan.dbt_skip_reason is not None) is test_case.expected_dbt_skipped assert (plan.sqlbuild_skip_reason is not None) is test_case.expected_sqlbuild_skipped From f17a8eddca8ee16b489056874b51ae68617c770b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:24:11 +0100 Subject: [PATCH 08/37] refactor: index dbt manifests and resolve dbt refs natively --- .../src/dbt_manifest/_helpers/fields.rs | 44 ++ .../src/dbt_manifest/_helpers/mod.rs | 1 + .../src/dbt_manifest/constants.rs | 27 + .../src/dbt_manifest/errors.rs | 97 +++ .../src/dbt_manifest/main/mod.rs | 3 + .../dbt_manifest/main/parse_manifest_model.rs | 56 ++ .../dbt_manifest/main/parse_manifest_seed.rs | 27 + .../main/parse_manifest_source.rs | 29 + .../sqlbuild-config/src/dbt_manifest/mod.rs | 8 + .../src/dbt_manifest/models.rs | 63 ++ .../sqlbuild-config/src/dbt_manifest/tests.rs | 6 + .../src/dbt_manifest/tests/helpers.rs | 16 + .../dbt_manifest/tests/test_dbt_manifest.rs | 59 ++ .../src/dbt_manifest/tests/test_types.rs | 8 + crates/sqlbuild-config/src/lib.rs | 1 + .../bindings/_helpers/dbt_interop/manifest.rs | 594 ++++++++++++++++++ .../src/bindings/_helpers/dbt_interop/mod.rs | 1 + .../_helpers/dbt_interop/registration.rs | 3 +- src/sqlbuild/_native.pyi | 32 + .../integrations/dbt/_helpers/graph/core.py | 5 +- .../dbt/_helpers/manifest/compile_refs.py | 78 --- .../dbt/_helpers/manifest/core.py | 312 --------- .../dbt/_helpers/manifest/sqlbuild_refs.py | 60 -- .../dbt/_helpers/pipeline/interop_prologue.py | 9 +- .../dbt/_helpers/planning/orchestration.py | 5 +- .../classes/dbt_compile_reference_resolver.py | 18 +- .../manifest/_build_compile_manifest_index.py | 14 - .../dbt/main/manifest/_load_manifest_index.py | 16 - .../_manifest_model_materialization.py | 12 - .../manifest/_resolve_reference_relation.py | 23 - .../manifest/_resolve_sqlbuild_model_refs.py | 20 - .../manifest/_validate_compile_model_names.py | 19 - .../_validate_compile_model_reference.py | 26 - .../build_compile_reference_resolver.py | 6 +- .../sqlbuild/integrations/dbt/test_graph.py | 8 +- .../planner/_helpers/resolve/test_refs.py | 2 +- .../sqlbuild/integrations/dbt/test_graph.py | 2 +- .../integrations/dbt/test_manifest.py | 4 +- .../dbt/test_plan_orchestration.py | 2 +- .../integrations/dbt/test_selection.py | 2 +- 40 files changed, 1102 insertions(+), 616 deletions(-) create mode 100644 crates/sqlbuild-config/src/dbt_manifest/_helpers/fields.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/_helpers/mod.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/constants.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/errors.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/main/mod.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_model.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_seed.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_source.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/mod.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/models.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/tests.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/tests/helpers.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/tests/test_dbt_manifest.rs create mode 100644 crates/sqlbuild-config/src/dbt_manifest/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/manifest/core.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_build_compile_manifest_index.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_load_manifest_index.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_manifest_model_materialization.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_resolve_reference_relation.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_names.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_reference.py diff --git a/crates/sqlbuild-config/src/dbt_manifest/_helpers/fields.rs b/crates/sqlbuild-config/src/dbt_manifest/_helpers/fields.rs new file mode 100644 index 0000000000..94656221c0 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/_helpers/fields.rs @@ -0,0 +1,44 @@ +//! Node field rules shared by models, sources and seeds. + +use crate::dbt_manifest::constants::DBT_REF_FUNCTION; +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::models::ManifestNode; + +/// A non-empty string field. +pub(crate) fn optional(node: &ManifestNode, field: &str) -> Option { + node.text + .iter() + .find(|(key, value)| key == field && !value.is_empty()) + .map(|(_, value)| value.clone()) +} + +/// A non-empty string field, or the invalid-manifest error naming it. +pub(crate) fn required(node: &ManifestNode, field: &str) -> Result { + optional(node, field) + .ok_or_else(|| ManifestError::invalid(&format!(": model {field} is required"))) +} + +/// The given relation name, or `database.schema.name` without empty parts. +pub(crate) fn relation_name( + node: &ManifestNode, + (database, schema): (&Option, &Option), + name: &str, +) -> String { + optional(node, "relation_name").unwrap_or_else(|| { + [database.as_deref(), schema.as_deref(), Some(name)] + .into_iter() + .flatten() + .filter(|part| !part.is_empty()) + .collect::>() + .join(".") + }) +} + +/// `SqlReferenceKind.DBT_REF.example_call(*args, quote='"')`. +pub(crate) fn quoted_dbt_ref(args: &[&str]) -> String { + let quoted: Vec = args + .iter() + .map(|arg| format!("\"{}\"", arg.replace('"', "\"\""))) + .collect(); + format!("{DBT_REF_FUNCTION}({})", quoted.join(", ")) +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/_helpers/mod.rs b/crates/sqlbuild-config/src/dbt_manifest/_helpers/mod.rs new file mode 100644 index 0000000000..f875d0dd21 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod fields; diff --git a/crates/sqlbuild-config/src/dbt_manifest/constants.rs b/crates/sqlbuild-config/src/dbt_manifest/constants.rs new file mode 100644 index 0000000000..4ec61bd069 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/constants.rs @@ -0,0 +1,27 @@ +//! dbt manifest resource types and the help SQLBuild shows for manifest problems. + +pub const MODEL_RESOURCE_TYPE: &str = "model"; +pub const SNAPSHOT_RESOURCE_TYPE: &str = "snapshot"; +pub const SEED_RESOURCE_TYPE: &str = "seed"; +pub const SOURCE_RESOURCE_TYPE: &str = "source"; +pub const TEST_RESOURCE_TYPE: &str = "test"; +pub const UNIT_TEST_RESOURCE_TYPE: &str = "unit_test"; +/// Node fields read as text: identity, namespace, relation and SQL. +pub const NODE_TEXT_FIELDS: [&str; 13] = [ + "package_name", + "name", + "source_name", + "database", + "schema", + "alias", + "identifier", + "relation_name", + "raw_code", + "compiled_code", + "raw_sql", + "compiled_sql", + "resource_type", +]; +pub(crate) const MANIFEST_HELP: &str = + "Run dbt compile or configure dbt target_path so SQLBuild can read manifest.json."; +pub(crate) const DBT_REF_FUNCTION: &str = "__dbt_ref"; diff --git a/crates/sqlbuild-config/src/dbt_manifest/errors.rs b/crates/sqlbuild-config/src/dbt_manifest/errors.rs new file mode 100644 index 0000000000..2aff1c52c3 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/errors.rs @@ -0,0 +1,97 @@ +//! A dbt manifest SQLBuild cannot read or resolve against. + +use crate::dbt_manifest::_helpers::fields::quoted_dbt_ref; +use crate::dbt_manifest::constants::{DBT_REF_FUNCTION, MANIFEST_HELP}; + +/// `CompileInputError(message, code=code, help=help)`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestError { + pub code: &'static str, + pub message: String, + pub help: Option, +} + +impl ManifestError { + /// No model `name` (in `package_name`, when given) exists in the manifest. + #[must_use] + pub fn unknown_model(name: &str, package_name: Option<&str>) -> Self { + let (target, what) = match package_name { + Some(package) => (format!("{package}.{name}"), "package/name"), + None => (name.to_owned(), "name"), + }; + Self { + code: "C212", + message: format!("dbt model '{target}' was not found in manifest"), + help: Some(format!( + "Check the {DBT_REF_FUNCTION}(...) {what} or run dbt compile to refresh manifest.json." + )), + } + } + + /// `name` matches models in several packages, listed sorted. + #[must_use] + pub fn ambiguous_model(name: &str, mut packages: Vec) -> Self { + packages.sort(); + Self { + code: "C213", + message: format!( + "dbt model '{name}' is ambiguous across packages: {}", + packages.join(", ") + ), + help: Some(format!( + "Use {} to choose one dbt model.", + quoted_dbt_ref(&["package_name", name]) + )), + } + } + + /// A model uses `__dbt_ref` but no manifest was found. + #[must_use] + pub fn missing_manifest(model_relative_path: &str, ref_name: &str) -> Self { + Self { + code: "C214", + message: format!( + "Model file {model_relative_path} uses {} but no dbt manifest was found", + quoted_dbt_ref(&[ref_name]) + ), + help: Some(MANIFEST_HELP.to_owned()), + } + } + + /// dbt and SQLBuild models share these names, sorted. + #[must_use] + pub fn shared_model_names(mut names: Vec) -> Option { + if names.is_empty() { + return None; + } + names.sort(); + Some(Self { + code: "C215", + message: format!("dbt and SQLBuild models share names: {}", names.join(", ")), + help: Some( + "Rename either the dbt model or SQLBuild model; owner-qualified names are not supported." + .to_owned(), + ), + }) + } + + /// The manifest file does not exist. + #[must_use] + pub fn missing_manifest_file(path: &str) -> Self { + Self { + code: "C210", + message: format!("dbt manifest file does not exist: {path}"), + help: Some(MANIFEST_HELP.to_owned()), + } + } + + /// The manifest is not the JSON shape SQLBuild reads. + #[must_use] + pub fn invalid(detail: &str) -> Self { + Self { + code: "C211", + message: format!("Invalid dbt manifest{detail}"), + help: None, + } + } +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/main/mod.rs b/crates/sqlbuild-config/src/dbt_manifest/main/mod.rs new file mode 100644 index 0000000000..e674181aa6 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/main/mod.rs @@ -0,0 +1,3 @@ +pub mod parse_manifest_model; +pub mod parse_manifest_seed; +pub mod parse_manifest_source; diff --git a/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_model.rs b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_model.rs new file mode 100644 index 0000000000..89ddd401a8 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_model.rs @@ -0,0 +1,56 @@ +//! A dbt model or snapshot node. + +use crate::dbt_manifest::_helpers::fields::{optional, relation_name, required}; +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::models::{ManifestNode, NodeChecksum, ParsedModel}; + +/// Every part when each is a non-empty string. +fn fqn_parts(parts: Vec>) -> Option> { + let mut names: Vec = Vec::new(); + for part in parts { + names.push(part.filter(|part| !part.is_empty())?); + } + Some(names) +} + +/// The model's identity, namespace, relation, checksum, fqn, query SQL and dependencies. +pub fn parse_manifest_model(node: &ManifestNode) -> Result { + let package_name: String = required(node, "package_name")?; + let name: String = required(node, "name")?; + let database: Option = optional(node, "database"); + let schema: Option = optional(node, "schema"); + let alias: Option = optional(node, "alias"); + let relation: String = relation_name( + node, + (&database, &schema), + alias.as_deref().unwrap_or(&name), + ); + let fqn: Vec = node.fqn.clone().and_then(fqn_parts).unwrap_or_default(); + let query_sql: String = ["raw_code", "compiled_code", "raw_sql", "compiled_sql"] + .into_iter() + .find_map(|field| optional(node, field)) + .unwrap_or_default(); + let node_checksum: Option = match &node.checksum { + NodeChecksum::Text(text) | NodeChecksum::Nested(Some(text)) if !text.is_empty() => { + Some(text.clone()) + } + _ => None, + }; + Ok(ParsedModel { + package_name, + name, + database, + schema, + alias, + node_checksum, + relation_name: relation, + fqn, + query_sql, + depends_on: node + .depends_on + .iter() + .filter(|unique_id| !unique_id.is_empty()) + .cloned() + .collect(), + }) +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_seed.rs b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_seed.rs new file mode 100644 index 0000000000..873bec4935 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_seed.rs @@ -0,0 +1,27 @@ +//! A dbt seed node. + +use crate::dbt_manifest::_helpers::fields::{optional, relation_name, required}; +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::models::{ManifestNode, ParsedSeed}; + +/// The seed's identity, namespace and relation. +pub fn parse_manifest_seed(node: &ManifestNode) -> Result { + let package_name: String = required(node, "package_name")?; + let name: String = required(node, "name")?; + let database: Option = optional(node, "database"); + let schema: Option = optional(node, "schema"); + let alias: Option = optional(node, "alias"); + let relation: String = relation_name( + node, + (&database, &schema), + alias.as_deref().unwrap_or(&name), + ); + Ok(ParsedSeed { + package_name, + name, + database, + schema, + alias, + relation_name: relation, + }) +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_source.rs b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_source.rs new file mode 100644 index 0000000000..e6a567aa1f --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_source.rs @@ -0,0 +1,29 @@ +//! A dbt source node. + +use crate::dbt_manifest::_helpers::fields::{optional, relation_name, required}; +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::models::{ManifestNode, ParsedSource}; + +/// The source's identity, namespace and relation. +pub fn parse_manifest_source(node: &ManifestNode) -> Result { + let package_name: String = required(node, "package_name")?; + let source_name: String = required(node, "source_name")?; + let name: String = required(node, "name")?; + let database: Option = optional(node, "database"); + let schema: Option = optional(node, "schema"); + let identifier: Option = optional(node, "identifier"); + let relation: String = relation_name( + node, + (&database, &schema), + identifier.as_deref().unwrap_or(&name), + ); + Ok(ParsedSource { + package_name, + source_name, + name, + database, + schema, + identifier, + relation_name: relation, + }) +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/mod.rs b/crates/sqlbuild-config/src/dbt_manifest/mod.rs new file mode 100644 index 0000000000..899237ebcb --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/dbt_manifest/models.rs b/crates/sqlbuild-config/src/dbt_manifest/models.rs new file mode 100644 index 0000000000..944ee3f3dd --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/models.rs @@ -0,0 +1,63 @@ +//! dbt manifest node fields and the parsed models, sources and seeds. + +/// A node `checksum` field. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum NodeChecksum { + /// A string value. + Text(String), + /// An object with this string `checksum` member, when it has one. + Nested(Option), + /// Missing or any other value. + #[default] + Other, +} + +/// The fields of one manifest node that SQLBuild reads. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ManifestNode { + /// `NODE_TEXT_FIELDS` whose value is a string, with that value. + pub text: Vec<(String, String)>, + pub checksum: NodeChecksum, + /// `fqn` when it is a list: each item when it is a string. + pub fqn: Option>>, + /// The string items of `depends_on.nodes`. + pub depends_on: Vec, +} + +/// A dbt model or snapshot node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParsedModel { + pub package_name: String, + pub name: String, + pub database: Option, + pub schema: Option, + pub alias: Option, + pub node_checksum: Option, + pub relation_name: String, + pub fqn: Vec, + pub query_sql: String, + pub depends_on: Vec, +} + +/// A dbt source node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParsedSource { + pub package_name: String, + pub source_name: String, + pub name: String, + pub database: Option, + pub schema: Option, + pub identifier: Option, + pub relation_name: String, +} + +/// A dbt seed node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParsedSeed { + pub package_name: String, + pub name: String, + pub database: Option, + pub schema: Option, + pub alias: Option, + pub relation_name: String, +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/tests.rs b/crates/sqlbuild-config/src/dbt_manifest/tests.rs new file mode 100644 index 0000000000..74248b83bd --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dbt_manifest.rs"] +mod dbt_manifest; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/dbt_manifest/tests/helpers.rs b/crates/sqlbuild-config/src/dbt_manifest/tests/helpers.rs new file mode 100644 index 0000000000..764e02e8a5 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/tests/helpers.rs @@ -0,0 +1,16 @@ +use crate::dbt_manifest::models::{ManifestNode, NodeChecksum}; + +pub(super) fn node(text: &[(&str, &str)], checksum: NodeChecksum) -> ManifestNode { + ManifestNode { + text: text + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(), + checksum, + fqn: Some(vec![ + Some("analytics".to_owned()), + Some("orders".to_owned()), + ]), + depends_on: vec!["model.a.b".to_owned(), String::new()], + } +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/tests/test_dbt_manifest.rs b/crates/sqlbuild-config/src/dbt_manifest/tests/test_dbt_manifest.rs new file mode 100644 index 0000000000..1557b19457 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/tests/test_dbt_manifest.rs @@ -0,0 +1,59 @@ +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::main::parse_manifest_model::parse_manifest_model; +use crate::dbt_manifest::models::{NodeChecksum, ParsedModel}; +use crate::dbt_manifest::tests::helpers::node; +use crate::dbt_manifest::tests::test_types::ModelTestCase; + +#[test] +fn given_manifest_model_nodes_when_parsing_then_matches_python_rules() { + let test_cases = [ + ModelTestCase { + description: "relation from namespace and alias, nested checksum, compiled sql fallback", + node: node( + &[ + ("package_name", "analytics"), + ("name", "orders"), + ("database", ""), + ("schema", "main"), + ("alias", "orders_v2"), + ("raw_code", ""), + ("compiled_code", "select 1"), + ], + NodeChecksum::Nested(Some("abc".to_owned())), + ), + expected_result: Ok(ParsedModel { + package_name: "analytics".to_owned(), + name: "orders".to_owned(), + database: None, + schema: Some("main".to_owned()), + alias: Some("orders_v2".to_owned()), + node_checksum: Some("abc".to_owned()), + relation_name: "main.orders_v2".to_owned(), + fqn: vec!["analytics".to_owned(), "orders".to_owned()], + query_sql: "select 1".to_owned(), + depends_on: vec!["model.a.b".to_owned()], + }), + }, + ModelTestCase { + description: "empty name is rejected", + node: node( + &[("package_name", "analytics"), ("name", "")], + NodeChecksum::Other, + ), + expected_result: Err(ManifestError { + code: "C211", + message: "Invalid dbt manifest: model name is required".to_owned(), + help: None, + }), + }, + ]; + + for test_case in test_cases { + assert_eq!( + parse_manifest_model(&test_case.node), + test_case.expected_result, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/tests/test_types.rs b/crates/sqlbuild-config/src/dbt_manifest/tests/test_types.rs new file mode 100644 index 0000000000..712d99f625 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::models::{ManifestNode, ParsedModel}; + +pub(super) struct ModelTestCase { + pub(super) description: &'static str, + pub(super) node: ManifestNode, + pub(super) expected_result: Result, +} diff --git a/crates/sqlbuild-config/src/lib.rs b/crates/sqlbuild-config/src/lib.rs index 504ad122f5..da69b29e36 100644 --- a/crates/sqlbuild-config/src/lib.rs +++ b/crates/sqlbuild-config/src/lib.rs @@ -4,6 +4,7 @@ pub mod command_connection; pub(crate) mod constants; +pub mod dbt_manifest; pub mod dbt_profile; pub mod effective; pub mod errors; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs new file mode 100644 index 0000000000..31f7a842d3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs @@ -0,0 +1,594 @@ +//! dbt manifest loading, indexing and `__dbt_ref` resolution. + +use std::collections::HashSet; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::dbt_manifest::constants::{ + MODEL_RESOURCE_TYPE, NODE_TEXT_FIELDS, SEED_RESOURCE_TYPE, SNAPSHOT_RESOURCE_TYPE, + SOURCE_RESOURCE_TYPE, TEST_RESOURCE_TYPE, UNIT_TEST_RESOURCE_TYPE, +}; +use sqlbuild_config::dbt_manifest::errors::ManifestError; +use sqlbuild_config::dbt_manifest::main::parse_manifest_model::parse_manifest_model; +use sqlbuild_config::dbt_manifest::main::parse_manifest_seed::parse_manifest_seed; +use sqlbuild_config::dbt_manifest::main::parse_manifest_source::parse_manifest_source; +use sqlbuild_config::dbt_manifest::models::{ + ManifestNode, NodeChecksum, ParsedModel, ParsedSeed, ParsedSource, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const DBT_REF_KIND: &str = "dbt_ref"; +const RELATION_OPTIONS: (&[&str], &[&str]) = + (&["manifest", "ref_kind", "ref_name", "ref_package"], &[]); + +fn raised(py: Python<'_>, error: ManifestError) -> PyErr { + let created = || -> PyResult { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", error.code)?; + keywords.set_item("help", error.help)?; + let instance: Bound<'_, PyAny> = imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")? + .call((error.message,), Some(&keywords))?; + Ok(PyErr::from_value(instance)) + }; + created().unwrap_or_else(|failure| failure) +} + +fn text_value(value: Option>) -> PyResult> { + match value { + Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), + _ => Ok(None), + } +} + +fn checksum(value: Option>) -> PyResult { + let Some(value) = value else { + return Ok(NodeChecksum::Other); + }; + if value.is_instance_of::() { + return Ok(NodeChecksum::Text(value.extract()?)); + } + match value.cast::() { + Ok(nested) => Ok(NodeChecksum::Nested(text_value( + nested.get_item("checksum")?, + )?)), + Err(_) => Ok(NodeChecksum::Other), + } +} + +fn fqn(value: Option>) -> PyResult>>> { + let Some(value) = value.filter(|value| value.is_instance_of::()) else { + return Ok(None); + }; + let mut parts: Vec> = Vec::new(); + for part in value.try_iter()? { + parts.push(text_value(Some(part?))?); + } + Ok(Some(parts)) +} + +fn depends_on(value: Option>) -> PyResult> { + let Some(Ok(depends_on)) = value.map(|value| value.cast_into::()) else { + return Ok(Vec::new()); + }; + let Some(nodes) = depends_on + .get_item("nodes")? + .filter(|nodes| nodes.is_instance_of::()) + else { + return Ok(Vec::new()); + }; + let mut unique_ids: Vec = Vec::new(); + for node in nodes.try_iter()? { + if let Some(unique_id) = text_value(Some(node?))? { + unique_ids.push(unique_id); + } + } + Ok(unique_ids) +} + +fn manifest_node(raw: &Bound<'_, PyDict>) -> PyResult { + let mut text: Vec<(String, String)> = Vec::new(); + for field in NODE_TEXT_FIELDS { + if let Some(value) = text_value(raw.get_item(field)?)? { + text.push((field.to_owned(), value)); + } + } + Ok(ManifestNode { + text, + checksum: checksum(raw.get_item("checksum")?)?, + fqn: fqn(raw.get_item("fqn")?)?, + depends_on: depends_on(raw.get_item("depends_on")?)?, + }) +} + +fn resource_type(raw: &Bound<'_, PyDict>) -> PyResult> { + text_value(raw.get_item("resource_type")?) +} + +/// `{str(key): value for key, value in raw_node.items()}`. +fn payload<'py>(raw: &Bound<'py, PyDict>) -> PyResult> { + let payload: Bound<'py, PyDict> = PyDict::new(raw.py()); + for (key, value) in raw.iter() { + payload.set_item(key.str()?, value)?; + } + Ok(payload.into_any()) +} + +fn optional<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { + match value { + Some(text) => PyString::new(py, text).into_any(), + None => py.None().into_bound(py), + } +} + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn model_object<'py>( + unique_id: &Bound<'py, PyAny>, + model: &ParsedModel, + raw: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = raw.py(); + call_with( + &imported(py, DBT_MODELS, "DbtManifestModel")?, + &[ + ("unique_id", unique_id.clone()), + ("package_name", text(py, &model.package_name)), + ("name", text(py, &model.name)), + ("database", optional(py, model.database.as_deref())), + ("schema", optional(py, model.schema.as_deref())), + ("alias", optional(py, model.alias.as_deref())), + ( + "node_checksum", + optional(py, model.node_checksum.as_deref()), + ), + ("relation_name", text(py, &model.relation_name)), + ("fqn", PyTuple::new(py, &model.fqn)?.into_any()), + ("query_sql", text(py, &model.query_sql)), + ( + "depends_on_nodes", + PyTuple::new(py, &model.depends_on)?.into_any(), + ), + ("payload", payload(raw)?), + ], + ) +} + +fn seed_object<'py>( + unique_id: &Bound<'py, PyAny>, + seed: &ParsedSeed, + raw: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = raw.py(); + call_with( + &imported(py, DBT_MODELS, "DbtManifestSeed")?, + &[ + ("unique_id", unique_id.clone()), + ("package_name", text(py, &seed.package_name)), + ("name", text(py, &seed.name)), + ("database", optional(py, seed.database.as_deref())), + ("schema", optional(py, seed.schema.as_deref())), + ("alias", optional(py, seed.alias.as_deref())), + ("relation_name", text(py, &seed.relation_name)), + ("payload", payload(raw)?), + ], + ) +} + +fn source_object<'py>( + unique_id: &Bound<'py, PyAny>, + source: &ParsedSource, + raw: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = raw.py(); + call_with( + &imported(py, DBT_MODELS, "DbtManifestSource")?, + &[ + ("unique_id", unique_id.clone()), + ("package_name", text(py, &source.package_name)), + ("source_name", text(py, &source.source_name)), + ("name", text(py, &source.name)), + ("database", optional(py, source.database.as_deref())), + ("schema", optional(py, source.schema.as_deref())), + ("identifier", optional(py, source.identifier.as_deref())), + ("relation_name", text(py, &source.relation_name)), + ("payload", payload(raw)?), + ], + ) +} + +/// The manifest section `key`: an object, or empty when missing and `optional`. +fn section<'py>( + manifest: &Bound<'py, PyDict>, + (key, optional_section): (&str, bool), +) -> PyResult> { + let py: Python<'py> = manifest.py(); + match manifest.get_item(key)? { + None if optional_section => Ok(PyDict::new(py)), + Some(value) if value.is_none() && optional_section => Ok(PyDict::new(py)), + Some(value) => value.cast_into::().map_err(|_| { + raised( + py, + ManifestError::invalid(&format!(": {key} must be an object")), + ) + }), + None => Err(raised( + py, + ManifestError::invalid(&format!(": {key} must be an object")), + )), + } +} + +/// `(unique_id, node)` pairs whose key is a string and value an object. +fn node_items<'py>(nodes: &Bound<'py, PyDict>) -> Vec<(Bound<'py, PyAny>, Bound<'py, PyDict>)> { + let mut items: Vec<(Bound<'py, PyAny>, Bound<'py, PyDict>)> = Vec::new(); + for (key, value) in nodes.iter() { + if !key.is_instance_of::() { + continue; + } + if let Ok(node) = value.cast_into::() { + items.push((key, node)); + } + } + items +} + +/// The model indexes built from `nodes`, in manifest order. +struct ModelIndexes<'py> { + by_unique_id: Bound<'py, PyDict>, + by_name: Bound<'py, PyDict>, + by_package_and_name: Bound<'py, PyDict>, + seeds: Bound<'py, PyDict>, +} + +fn index_nodes<'py>(nodes: &Bound<'py, PyDict>) -> PyResult> { + let py: Python<'py> = nodes.py(); + let indexes: ModelIndexes<'py> = ModelIndexes { + by_unique_id: PyDict::new(py), + by_name: PyDict::new(py), + by_package_and_name: PyDict::new(py), + seeds: PyDict::new(py), + }; + for (unique_id, raw) in node_items(nodes) { + let kind: Option = resource_type(&raw)?; + let kind: &str = kind.as_deref().unwrap_or_default(); + if kind == MODEL_RESOURCE_TYPE || kind == SNAPSHOT_RESOURCE_TYPE { + let model: ParsedModel = + parse_manifest_model(&manifest_node(&raw)?).map_err(|error| raised(py, error))?; + let object: Bound<'py, PyAny> = model_object(&unique_id, &model, &raw)?; + indexes.by_unique_id.set_item(&unique_id, &object)?; + let named: Bound<'py, PyAny> = match indexes.by_name.get_item(&model.name)? { + Some(named) => named, + None => { + let named: Bound<'py, PyList> = PyList::empty(py); + indexes.by_name.set_item(&model.name, &named)?; + named.into_any() + } + }; + named.call_method1("append", (&object,))?; + indexes + .by_package_and_name + .set_item((&model.package_name, &model.name), &object)?; + } else if kind == SEED_RESOURCE_TYPE { + let seed: ParsedSeed = + parse_manifest_seed(&manifest_node(&raw)?).map_err(|error| raised(py, error))?; + indexes + .seeds + .set_item(&unique_id, seed_object(&unique_id, &seed, &raw)?)?; + } + } + let named: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = indexes.by_name.iter().collect(); + for (name, models) in named { + let models: Vec> = models.try_iter()?.collect::>()?; + indexes.by_name.set_item(name, PyTuple::new(py, models)?)?; + } + Ok(indexes) +} + +fn validation_dependencies<'py>( + sections: [&Bound<'py, PyDict>; 2], +) -> PyResult> { + let py: Python<'py> = sections[0].py(); + let dependencies: Bound<'py, PyDict> = PyDict::new(py); + for nodes in sections { + for (unique_id, raw) in node_items(nodes) { + let kind: Option = resource_type(&raw)?; + if kind.as_deref() != Some(TEST_RESOURCE_TYPE) + && kind.as_deref() != Some(UNIT_TEST_RESOURCE_TYPE) + { + continue; + } + let unique_ids: Vec = depends_on(raw.get_item("depends_on")?)? + .into_iter() + .filter(|node| !node.is_empty()) + .collect(); + dependencies.set_item(unique_id, PyTuple::new(py, unique_ids)?)?; + } + } + Ok(dependencies) +} + +fn manifest_index<'py>(raw_data: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = raw_data.py(); + let manifest: &Bound<'py, PyDict> = raw_data + .cast::() + .map_err(|_| raised(py, ManifestError::invalid(": root must be an object")))?; + let nodes: Bound<'py, PyDict> = section(manifest, ("nodes", false))?; + let sources: Bound<'py, PyDict> = section(manifest, ("sources", true))?; + let unit_tests: Bound<'py, PyDict> = section(manifest, ("unit_tests", true))?; + let indexes: ModelIndexes<'py> = index_nodes(&nodes)?; + let validation: Bound<'py, PyDict> = validation_dependencies([&nodes, &unit_tests])?; + let sources_by_unique_id: Bound<'py, PyDict> = PyDict::new(py); + for (unique_id, raw) in node_items(&sources) { + if resource_type(&raw)?.as_deref() != Some(SOURCE_RESOURCE_TYPE) { + continue; + } + let source: ParsedSource = + parse_manifest_source(&manifest_node(&raw)?).map_err(|error| raised(py, error))?; + sources_by_unique_id.set_item(&unique_id, source_object(&unique_id, &source, &raw)?)?; + } + call_with( + &imported(py, DBT_MODELS, "DbtManifestIndex")?, + &[ + ("models_by_unique_id", indexes.by_unique_id.into_any()), + ("models_by_name", indexes.by_name.into_any()), + ( + "models_by_package_and_name", + indexes.by_package_and_name.into_any(), + ), + ("sources_by_unique_id", sources_by_unique_id.into_any()), + ("seeds_by_unique_id", indexes.seeds.into_any()), + ( + "validation_depends_on_nodes_by_unique_id", + validation.into_any(), + ), + ], + ) +} + +/// `json.loads(contents)`, with a decode failure raised as an invalid manifest. +fn decoded<'py>(contents: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = contents.py(); + match imported(py, "json", "loads")?.call1((contents,)) { + Ok(value) => Ok(value), + Err(error) if error.is_instance(py, &imported(py, "json", "JSONDecodeError")?) => { + let detail: String = error.value(py).getattr("msg")?.extract()?; + Err(raised( + py, + ManifestError::invalid(&format!(" JSON: {detail}")), + )) + } + Err(error) => Err(error), + } +} + +/// Load and index dbt model nodes from a manifest.json file. +#[pyfunction] +#[pyo3(signature = (*, manifest_path))] +fn load_dbt_manifest_index<'py>( + py: Python<'py>, + manifest_path: Bound<'py, PyAny>, +) -> PyResult> { + if !manifest_path.call_method0("is_file")?.is_truthy()? { + let path: String = manifest_path.str()?.extract()?; + return Err(raised(py, ManifestError::missing_manifest_file(&path))); + } + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("encoding", "utf-8")?; + let contents: Bound<'py, PyAny> = + manifest_path.call_method("read_text", (), Some(&keywords))?; + manifest_index(&decoded(&contents)?) +} + +/// Build a dbt manifest index from discovered manifest contents. +#[pyfunction] +#[pyo3(signature = (*, manifest_contents))] +fn build_compile_dbt_manifest_index<'py>( + py: Python<'py>, + manifest_contents: Bound<'py, PyAny>, +) -> PyResult> { + if manifest_contents.is_none() { + return Ok(py.None().into_bound(py)); + } + manifest_index(&decoded(&manifest_contents)?) +} + +/// Build dbt model lookup indexes from decoded manifest JSON. +#[pyfunction] +#[pyo3(signature = (*, raw_data))] +fn build_dbt_manifest_index<'py>(raw_data: Bound<'py, PyAny>) -> PyResult> { + manifest_index(&raw_data) +} + +/// The manifest model a one-arg or package-qualified `__dbt_ref` names. +fn resolved_model<'py>( + manifest: &Bound<'py, PyAny>, + name: &str, + package_name: Option<&str>, +) -> PyResult> { + let py: Python<'py> = manifest.py(); + if let Some(package) = package_name { + let model: Bound<'py, PyAny> = manifest + .getattr("models_by_package_and_name")? + .call_method1("get", ((package, name),))?; + if model.is_none() { + return Err(raised( + py, + ManifestError::unknown_model(name, Some(package)), + )); + } + return Ok(model); + } + let matches: Bound<'py, PyAny> = manifest + .getattr("models_by_name")? + .call_method1("get", (name, PyTuple::empty(py)))?; + let matches: Vec> = matches.try_iter()?.collect::>()?; + match matches.as_slice() { + [] => Err(raised(py, ManifestError::unknown_model(name, None))), + [model] => Ok(model.clone()), + _ => { + let packages: Vec = matches + .iter() + .map(|model| model.getattr("package_name")?.extract()) + .collect::>()?; + Err(raised(py, ManifestError::ambiguous_model(name, packages))) + } + } +} + +/// Resolve a dbt model by one-arg or package-qualified ref semantics. +#[pyfunction] +#[pyo3(signature = (*, manifest, name, package_name = None))] +fn resolve_dbt_manifest_model<'py>( + manifest: Bound<'py, PyAny>, + name: String, + package_name: Option, +) -> PyResult> { + resolved_model(&manifest, &name, package_name.as_deref()) +} + +/// Reject ambiguous ownership between dbt and SQLBuild models. +#[pyfunction] +#[pyo3(signature = (*, known_model_names, dbt_manifest))] +fn validate_compile_dbt_model_names( + py: Python<'_>, + known_model_names: Bound<'_, PyAny>, + dbt_manifest: Bound<'_, PyAny>, +) -> Result<(), PyErr> { + if dbt_manifest.is_none() { + return Ok(()); + } + let dbt_names: Bound<'_, PyAny> = dbt_manifest.getattr("models_by_name")?; + let mut shared: Vec = Vec::new(); + for name in known_model_names.try_iter()? { + let name: Bound<'_, PyAny> = name?; + if dbt_names.contains(&name)? { + shared.push(name.extract()?); + } + } + match ManifestError::shared_model_names(shared) { + Some(error) => Err(raised(py, error)), + None => Ok(()), + } +} + +fn dbt_ref_kind(ref_kind: &Bound<'_, PyAny>) -> PyResult { + ref_kind.eq(DBT_REF_KIND) +} + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +/// Validate one compile-time dbt ref against a dbt manifest. +#[pyfunction] +#[pyo3(signature = (*, reference, model_relative_path, dbt_manifest))] +fn validate_compile_dbt_model_reference( + py: Python<'_>, + reference: Bound<'_, PyAny>, + model_relative_path: Bound<'_, PyAny>, + dbt_manifest: Bound<'_, PyAny>, +) -> Result<(), PyErr> { + if !dbt_ref_kind(&reference.getattr("ref_kind")?)? { + return Ok(()); + } + let name: String = reference.getattr("ref_name")?.extract()?; + if dbt_manifest.is_none() { + let path: String = model_relative_path.str()?.extract()?; + return Err(raised(py, ManifestError::missing_manifest(&path, &name))); + } + let package: Option = optional_text(&reference.getattr("ref_package")?)?; + resolved_model(&dbt_manifest, &name, package.as_deref())?; + Ok(()) +} + +fn name_set(names: &Bound<'_, PyAny>) -> PyResult> { + let mut set: HashSet = HashSet::new(); + for name in names.try_iter()? { + set.insert(name?.extract()?); + } + Ok(set) +} + +/// Return resolved dbt refs used directly by SQLBuild models. +#[pyfunction] +#[pyo3(signature = (*, project, manifest, selected_model_names = None))] +fn resolve_sqlbuild_model_dbt_refs<'py>( + py: Python<'py>, + project: Bound<'py, PyAny>, + manifest: Bound<'py, PyAny>, + selected_model_names: Option>, +) -> PyResult> { + let selected_model_names: Option> = selected_model_names + .map(|names| name_set(&names)) + .transpose()?; + let mut refs: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + if let Some(selected) = &selected_model_names { + let name: String = model.getattr("name")?.extract()?; + if !selected.contains(&name) { + continue; + } + } + for reference in model.getattr("references")?.try_iter()? { + let reference: Bound<'py, PyAny> = reference?; + if !dbt_ref_kind(&reference.getattr("ref_kind")?)? { + continue; + } + let name: String = reference.getattr("ref_name")?.extract()?; + let package: Option = optional_text(&reference.getattr("ref_package")?)?; + let resolved: Bound<'py, PyAny> = resolved_model(&manifest, &name, package.as_deref())?; + refs.push(PyTuple::new(py, [model.clone(), resolved])?); + } + } + PyTuple::new(py, refs) +} + +/// Resolve one external dbt reference to its manifest relation. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_dbt_reference_relation<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_dbt_reference_relation", + options, + RELATION_OPTIONS, + )?; + let manifest: Bound<'py, PyAny> = arguments.value("manifest")?; + if !dbt_ref_kind(&arguments.value("ref_kind")?)? || manifest.is_none() { + return Ok(py.None().into_bound(py)); + } + let name: String = arguments.value("ref_name")?.extract()?; + let package: Option = optional_text(&arguments.value("ref_package")?)?; + resolved_model(&manifest, &name, package.as_deref())?.getattr("relation_name") +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(load_dbt_manifest_index, module)?)?; + module.add_function(wrap_pyfunction!(build_compile_dbt_manifest_index, module)?)?; + module.add_function(wrap_pyfunction!(build_dbt_manifest_index, module)?)?; + module.add_function(wrap_pyfunction!(resolve_dbt_manifest_model, module)?)?; + module.add_function(wrap_pyfunction!(validate_compile_dbt_model_names, module)?)?; + module.add_function(wrap_pyfunction!( + validate_compile_dbt_model_reference, + module + )?)?; + module.add_function(wrap_pyfunction!(resolve_sqlbuild_model_dbt_refs, module)?)?; + module.add_function(wrap_pyfunction!(resolve_dbt_reference_relation, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs index 01385ad05a..40ad34c6e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod manifest; pub(crate) mod plan; pub(crate) mod profile; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index f2265bf19e..2cd3a2ed8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dbt_interop::{plan, profile}; +use crate::bindings::_helpers::dbt_interop::{manifest, plan, profile}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + manifest::register(module)?; plan::register(module)?; profile::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 34c2e0fb18..881fbb0072 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -230,6 +230,8 @@ from sqlbuild.integrations.dbt.models import ( DbtInteropPlan, DbtInteropSelectionResult, DbtLsNode, + DbtManifestIndex, + DbtManifestModel, NormalizedDbtProfileConnection, ResolvedDbtProfileOutput, ) @@ -1395,6 +1397,36 @@ def build_dbt_init_project_toml( target_schema: str | None, target_database: str | None, ) -> str: ... +def load_dbt_manifest_index(*, manifest_path: Path) -> DbtManifestIndex: ... +def build_compile_dbt_manifest_index( + *, manifest_contents: str | None +) -> DbtManifestIndex | None: ... +def build_dbt_manifest_index(*, raw_data: object) -> DbtManifestIndex: ... +def resolve_dbt_manifest_model( + *, manifest: DbtManifestIndex, name: str, package_name: str | None = None +) -> DbtManifestModel: ... +def validate_compile_dbt_model_names( + *, known_model_names: Iterable[str], dbt_manifest: DbtManifestIndex | None +) -> None: ... +def validate_compile_dbt_model_reference( + *, + reference: CompileSqlReference, + model_relative_path: Path, + dbt_manifest: DbtManifestIndex | None, +) -> None: ... +def resolve_sqlbuild_model_dbt_refs( + *, + project: CompiledProjectView, + manifest: DbtManifestIndex, + selected_model_names: Iterable[str] | None = None, +) -> tuple[tuple[CompiledModelView, DbtManifestModel], ...]: ... +def resolve_dbt_reference_relation( + *, + manifest: DbtManifestIndex | None, + ref_kind: str, + ref_name: str, + ref_package: str | None, +) -> str | None: ... def build_dbt_interop_plan( *, command: DbtInteropCommand | str, diff --git a/src/sqlbuild/integrations/dbt/_helpers/graph/core.py b/src/sqlbuild/integrations/dbt/_helpers/graph/core.py index 80290e1baf..3a6ff7d900 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/graph/core.py +++ b/src/sqlbuild/integrations/dbt/_helpers/graph/core.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild._native import invert_edges, transitive_closure +from sqlbuild._native import invert_edges, resolve_sqlbuild_model_dbt_refs, transitive_closure from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) @@ -11,9 +11,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( - resolve_sqlbuild_model_dbt_refs, -) from sqlbuild.integrations.dbt.models import ( DbtCombinedGraph, DbtCombinedGraphKey, diff --git a/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py b/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py deleted file mode 100644 index 77d0698d31..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Compile-time dbt ref helpers used by SQLBuild model attachment.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.integrations.dbt._helpers.manifest.core import ( - build_dbt_manifest_index, - resolve_dbt_manifest_model, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def build_compile_dbt_manifest_index(*, manifest_contents: str | None) -> DbtManifestIndex | None: - """Build a dbt manifest index from discovered manifest contents.""" - - if manifest_contents is None: - return None - try: - raw_data: object = json.loads(manifest_contents) - except json.JSONDecodeError as exc: - raise CompileInputError( - f"Invalid dbt manifest JSON: {exc.msg}", - code="C211", - ) from exc - return build_dbt_manifest_index(raw_data=raw_data) - - -def validate_compile_dbt_model_names( - *, known_model_names: set[str], dbt_manifest: DbtManifestIndex | None -) -> None: - """Reject ambiguous ownership between dbt and SQLBuild models.""" - - if dbt_manifest is None: - return - duplicate_names: tuple[str, ...] = tuple( - sorted(name for name in known_model_names if name in dbt_manifest.models_by_name) - ) - if duplicate_names: - raise CompileInputError( - f"dbt and SQLBuild models share names: {', '.join(duplicate_names)}", - code="C215", - help=( - "Rename either the dbt model or SQLBuild model; owner-qualified names " - "are not supported." - ), - ) - - -def validate_compile_dbt_model_reference( - *, - reference: CompileSqlReference, - model_relative_path: Path, - dbt_manifest: DbtManifestIndex | None, -) -> None: - """Validate one compile-time dbt ref against a dbt manifest.""" - - if reference.ref_kind != SqlReferenceKind.DBT_REF: - return - if dbt_manifest is None: - raise CompileInputError( - f"Model file {model_relative_path} uses " - f"{SqlReferenceKind.DBT_REF.example_call(reference.ref_name, quote='"')} but no dbt " - "manifest was found", - code="C214", - help=( - "Run dbt compile or configure dbt target_path so SQLBuild can read manifest.json." - ), - ) - resolve_dbt_manifest_model( - manifest=dbt_manifest, - package_name=reference.ref_package, - name=reference.ref_name, - ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/manifest/core.py b/src/sqlbuild/integrations/dbt/_helpers/manifest/core.py deleted file mode 100644 index e0bde28e89..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/manifest/core.py +++ /dev/null @@ -1,312 +0,0 @@ -"""dbt manifest loading and model lookup helpers.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, cast - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.integrations.dbt.constants import ( - DBT_MANIFEST_CONFIG_KEY, - DBT_MANIFEST_MATERIALIZED_KEY, -) -from sqlbuild.integrations.dbt.models import ( - DbtManifestIndex, - DbtManifestModel, - DbtManifestSeed, - DbtManifestSource, -) -from sqlbuild.integrations.dbt.types import DbtSupportedResourceType - -_INDEXED_NODE_RESOURCE_TYPES: frozenset[DbtSupportedResourceType] = frozenset( - {DbtSupportedResourceType.MODEL, DbtSupportedResourceType.SNAPSHOT} -) - - -def load_dbt_manifest_index(*, manifest_path: Path) -> DbtManifestIndex: - """Load and index dbt model nodes from a manifest.json file.""" - - if not manifest_path.is_file(): - raise CompileInputError( - f"dbt manifest file does not exist: {manifest_path}", - code="C210", - help="Run dbt compile or configure dbt target_path so SQLBuild can read manifest.json.", - ) - try: - raw_data: Any = json.loads(manifest_path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise CompileInputError( - f"Invalid dbt manifest JSON: {exc.msg}", - code="C211", - ) from exc - return build_dbt_manifest_index(raw_data=raw_data) - - -def build_dbt_manifest_index(*, raw_data: object) -> DbtManifestIndex: - """Build dbt model lookup indexes from decoded manifest JSON.""" - - if not isinstance(raw_data, dict): - raise CompileInputError("Invalid dbt manifest: root must be an object", code="C211") - manifest_data: dict[str, object] = cast(dict[str, object], raw_data) - raw_nodes: object | None = manifest_data.get("nodes") - if not isinstance(raw_nodes, dict): - raise CompileInputError("Invalid dbt manifest: nodes must be an object", code="C211") - raw_sources: object | None = manifest_data.get("sources") - if raw_sources is None: - raw_sources = {} - if not isinstance(raw_sources, dict): - raise CompileInputError("Invalid dbt manifest: sources must be an object", code="C211") - raw_unit_tests: object | None = manifest_data.get("unit_tests") - if raw_unit_tests is None: - raw_unit_tests = {} - if not isinstance(raw_unit_tests, dict): - raise CompileInputError("Invalid dbt manifest: unit_tests must be an object", code="C211") - - models_by_unique_id: dict[str, DbtManifestModel] = {} - models_by_name_lists: dict[str, list[DbtManifestModel]] = {} - models_by_package_and_name: dict[tuple[str, str], DbtManifestModel] = {} - sources_by_unique_id: dict[str, DbtManifestSource] = {} - seeds_by_unique_id: dict[str, DbtManifestSeed] = {} - validation_depends_on_nodes_by_unique_id: dict[str, tuple[str, ...]] = {} - unique_id: object - raw_node: object - for unique_id, raw_node in raw_nodes.items(): - if not isinstance(unique_id, str) or not isinstance(raw_node, dict): - continue - node_data: dict[object, object] = cast(dict[object, object], raw_node) - resource_type: object = node_data.get("resource_type") - if resource_type in _INDEXED_NODE_RESOURCE_TYPES: - model: DbtManifestModel = _parse_model(unique_id=unique_id, raw_node=node_data) - models_by_unique_id[model.unique_id] = model - models_by_name_lists.setdefault(model.name, []).append(model) - models_by_package_and_name[(model.package_name, model.name)] = model - continue - if resource_type == DbtSupportedResourceType.SEED: - seed: DbtManifestSeed = _parse_seed(unique_id=unique_id, raw_node=node_data) - seeds_by_unique_id[seed.unique_id] = seed - continue - raw_validation_nodes: dict[object, object] - for raw_validation_nodes in ( - cast(dict[object, object], raw_nodes), - cast(dict[object, object], raw_unit_tests), - ): - for unique_id, raw_node in raw_validation_nodes.items(): - if not isinstance(unique_id, str) or not isinstance(raw_node, dict): - continue - node_data = cast(dict[object, object], raw_node) - if node_data.get("resource_type") not in ( - DbtSupportedResourceType.TEST, - DbtSupportedResourceType.UNIT_TEST, - ): - continue - validation_depends_on_nodes_by_unique_id[unique_id] = _parse_depends_on_nodes( - node_data.get("depends_on") - ) - - for unique_id, raw_node in raw_sources.items(): - if not isinstance(unique_id, str) or not isinstance(raw_node, dict): - continue - node_data = cast(dict[object, object], raw_node) - if node_data.get("resource_type") != DbtSupportedResourceType.SOURCE: - continue - source: DbtManifestSource = _parse_source(unique_id=unique_id, raw_node=node_data) - sources_by_unique_id[source.unique_id] = source - - return DbtManifestIndex( - models_by_unique_id=models_by_unique_id, - models_by_name={name: tuple(models) for name, models in models_by_name_lists.items()}, - models_by_package_and_name=models_by_package_and_name, - sources_by_unique_id=sources_by_unique_id, - seeds_by_unique_id=seeds_by_unique_id, - validation_depends_on_nodes_by_unique_id=validation_depends_on_nodes_by_unique_id, - ) - - -def resolve_dbt_manifest_model( - *, manifest: DbtManifestIndex, name: str, package_name: str | None = None -) -> DbtManifestModel: - """Resolve a dbt model by one-arg or package-qualified ref semantics.""" - - if package_name is not None: - model: DbtManifestModel | None = manifest.models_by_package_and_name.get( - (package_name, name) - ) - if model is None: - raise CompileInputError( - f"dbt model '{package_name}.{name}' was not found in manifest", - code="C212", - help=( - f"Check the {SqlReferenceKind.DBT_REF.placeholder_call('...')} package/name " - "or run dbt compile to refresh manifest.json." - ), - ) - return model - - matches: tuple[DbtManifestModel, ...] = manifest.models_by_name.get(name, ()) - if not matches: - raise CompileInputError( - f"dbt model '{name}' was not found in manifest", - code="C212", - help=( - f"Check the {SqlReferenceKind.DBT_REF.placeholder_call('...')} name or run dbt " - "compile to refresh manifest.json." - ), - ) - if len(matches) > 1: - packages: str = ", ".join(sorted(model.package_name for model in matches)) - raise CompileInputError( - f"dbt model '{name}' is ambiguous across packages: {packages}", - code="C213", - help=( - f"Use {SqlReferenceKind.DBT_REF.example_call('package_name', name, quote='"')} to " - "choose one dbt model." - ), - ) - return matches[0] - - -def dbt_manifest_model_materialization(*, model: DbtManifestModel) -> str | None: - """Return the lowercased dbt materialization for a manifest model, if declared.""" - - config: object | None = model.payload.get(DBT_MANIFEST_CONFIG_KEY) - if not isinstance(config, dict): - return None - materialized: object | None = cast(dict[str, object], config).get(DBT_MANIFEST_MATERIALIZED_KEY) - if not isinstance(materialized, str) or not materialized.strip(): - return None - return materialized.strip().lower() - - -def _parse_model(*, unique_id: str, raw_node: dict[object, object]) -> DbtManifestModel: - package_name: str = _required_str(value=raw_node.get("package_name"), field_name="package_name") - name: str = _required_str(value=raw_node.get("name"), field_name="name") - database: str | None = _optional_str(raw_node.get("database")) - schema: str | None = _optional_str(raw_node.get("schema")) - alias: str | None = _optional_str(raw_node.get("alias")) - relation_name: str | None = _optional_str(raw_node.get("relation_name")) - if relation_name is None: - relation_name = _render_relation_name(database=database, schema=schema, name=alias or name) - depends_on_nodes: tuple[str, ...] = _parse_depends_on_nodes(raw_node.get("depends_on")) - return DbtManifestModel( - unique_id=unique_id, - package_name=package_name, - name=name, - database=database, - schema=schema, - alias=alias, - node_checksum=_parse_checksum(raw_node.get("checksum")), - relation_name=relation_name, - fqn=_parse_fqn(raw_node.get("fqn")), - query_sql=_model_query_sql(raw_node=raw_node), - depends_on_nodes=depends_on_nodes, - payload={str(key): value for key, value in raw_node.items()}, - ) - - -def _parse_source(*, unique_id: str, raw_node: dict[object, object]) -> DbtManifestSource: - package_name: str = _required_str(value=raw_node.get("package_name"), field_name="package_name") - source_name: str = _required_str(value=raw_node.get("source_name"), field_name="source_name") - name: str = _required_str(value=raw_node.get("name"), field_name="name") - database: str | None = _optional_str(raw_node.get("database")) - schema: str | None = _optional_str(raw_node.get("schema")) - identifier: str | None = _optional_str(raw_node.get("identifier")) - relation_name: str | None = _optional_str(raw_node.get("relation_name")) - if relation_name is None: - relation_name = _render_relation_name( - database=database, - schema=schema, - name=identifier or name, - ) - return DbtManifestSource( - unique_id=unique_id, - package_name=package_name, - source_name=source_name, - name=name, - database=database, - schema=schema, - identifier=identifier, - relation_name=relation_name, - payload={str(key): value for key, value in raw_node.items()}, - ) - - -def _parse_seed(*, unique_id: str, raw_node: dict[object, object]) -> DbtManifestSeed: - package_name: str = _required_str(value=raw_node.get("package_name"), field_name="package_name") - name: str = _required_str(value=raw_node.get("name"), field_name="name") - database: str | None = _optional_str(raw_node.get("database")) - schema: str | None = _optional_str(raw_node.get("schema")) - alias: str | None = _optional_str(raw_node.get("alias")) - relation_name: str | None = _optional_str(raw_node.get("relation_name")) - if relation_name is None: - relation_name = _render_relation_name(database=database, schema=schema, name=alias or name) - return DbtManifestSeed( - unique_id=unique_id, - package_name=package_name, - name=name, - database=database, - schema=schema, - alias=alias, - relation_name=relation_name, - payload={str(key): value for key, value in raw_node.items()}, - ) - - -def _render_relation_name(*, database: str | None, schema: str | None, name: str) -> str: - parts: tuple[str, ...] = tuple(part for part in (database, schema, name) if part) - return ".".join(parts) - - -def _model_query_sql(*, raw_node: dict[object, object]) -> str: - return ( - _optional_str(raw_node.get("raw_code")) - or _optional_str(raw_node.get("compiled_code")) - or _optional_str(raw_node.get("raw_sql")) - or _optional_str(raw_node.get("compiled_sql")) - or "" - ) - - -def _required_str(*, value: object, field_name: str) -> str: - if isinstance(value, str) and value: - return value - raise CompileInputError(f"Invalid dbt manifest: model {field_name} is required", code="C211") - - -def _optional_str(value: object) -> str | None: - if isinstance(value, str) and value: - return value - return None - - -def _parse_fqn(value: object) -> tuple[str, ...]: - if not isinstance(value, list): - return () - parts: list[str] = [] - item: object - for item in value: - if not isinstance(item, str) or not item: - return () - parts.append(item) - return tuple(parts) - - -def _parse_checksum(value: object) -> str | None: - if isinstance(value, str) and value: - return value - if not isinstance(value, dict): - return None - raw_checksum: dict[str, object] = cast(dict[str, object], value) - checksum: object | None = raw_checksum.get("checksum") - return checksum if isinstance(checksum, str) and checksum else None - - -def _parse_depends_on_nodes(value: object) -> tuple[str, ...]: - if not isinstance(value, dict): - return () - depends_on: dict[str, object] = cast(dict[str, object], value) - raw_nodes: object | None = depends_on.get("nodes") - if not isinstance(raw_nodes, list): - return () - return tuple(node for node in raw_nodes if isinstance(node, str) and node) diff --git a/src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py b/src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py deleted file mode 100644 index 3a63266d32..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Helpers for SQLBuild model references into dbt manifests.""" - -from __future__ import annotations - -from collections.abc import Sequence - -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.integrations.dbt._helpers.manifest.core import resolve_dbt_manifest_model -from sqlbuild.integrations.dbt.models import DbtManifestIndex, DbtManifestModel - - -def resolve_sqlbuild_model_dbt_refs( - *, - project: CompiledProjectView, - manifest: DbtManifestIndex, - selected_model_names: Sequence[str] | None = None, -) -> tuple[tuple[CompiledModelView, DbtManifestModel], ...]: - """Return resolved dbt refs used directly by SQLBuild models.""" - - selected_names: frozenset[str] | None = ( - None if selected_model_names is None else frozenset(selected_model_names) - ) - refs: list[tuple[CompiledModelView, DbtManifestModel]] = [] - model: CompiledModelView - for model in project.models: - if selected_names is not None and model.name not in selected_names: - continue - for reference in model.references: - if reference.ref_kind != SqlReferenceKind.DBT_REF: - continue - refs.append( - ( - model, - resolve_dbt_manifest_model( - manifest=manifest, - package_name=reference.ref_package, - name=reference.ref_name, - ), - ) - ) - return tuple(refs) - - -def resolve_dbt_reference_relation( - *, - manifest: DbtManifestIndex | None, - ref_kind: str, - ref_name: str, - ref_package: str | None, -) -> str | None: - """Resolve one external dbt reference to its manifest relation.""" - - if ref_kind != SqlReferenceKind.DBT_REF or manifest is None: - return None - return resolve_dbt_manifest_model( - manifest=manifest, - package_name=ref_package, - name=ref_name, - ).relation_name diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index d25ac65540..20c24aa6fd 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -8,7 +8,11 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + discover_project_inputs, + load_dbt_manifest_index, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -24,9 +28,6 @@ from sqlbuild.integrations.dbt.main.config._resolve_plan_options import resolve_dbt_plan_options from sqlbuild.integrations.dbt.main.config._resolve_vars_mapping import resolve_dbt_vars_mapping from sqlbuild.integrations.dbt.main.graph._build_combined_graph import build_dbt_combined_graph -from sqlbuild.integrations.dbt.main.manifest._load_manifest_index import ( - load_manifest_index as load_dbt_manifest_index, -) from sqlbuild.integrations.dbt.main.manifest._resolve_manifest_path import ( resolve_dbt_manifest_path, ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py index e9c9b55707..f2d0be7060 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py +++ b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py @@ -4,16 +4,13 @@ from collections.abc import Sequence -from sqlbuild._native import build_dbt_interop_plan +from sqlbuild._native import build_dbt_interop_plan, resolve_sqlbuild_model_dbt_refs from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion from sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors import ( resolve_sqlbuild_model_selector_names, ) from sqlbuild.compiler.planner.models import SelectorExpansion -from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( - resolve_sqlbuild_model_dbt_refs, -) from sqlbuild.integrations.dbt._helpers.selection.core import resolve_dbt_interop_sqlbuild_selection from sqlbuild.integrations.dbt._helpers.selection.selector_terms import dbt_fqn_selector_term from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner diff --git a/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py b/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py index 9043efdb0f..fd2892daf4 100644 --- a/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py +++ b/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py @@ -2,17 +2,13 @@ from pathlib import Path -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.integrations.dbt.main.manifest._resolve_reference_relation import ( +from sqlbuild._native import ( resolve_dbt_reference_relation, + validate_compile_dbt_model_names, + validate_compile_dbt_model_reference, ) -from sqlbuild.integrations.dbt.main.manifest._validate_compile_model_names import ( - validate_compile_model_names, -) -from sqlbuild.integrations.dbt.main.manifest._validate_compile_model_reference import ( - validate_compile_model_reference, -) +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.integrations.dbt.models import DbtManifestIndex @@ -23,7 +19,7 @@ def __init__(self, *, dbt_manifest: DbtManifestIndex | None) -> None: self._dbt_manifest = dbt_manifest def validate_model_names(self, *, known_model_names: set[str]) -> None: - validate_compile_model_names( + validate_compile_dbt_model_names( known_model_names=known_model_names, dbt_manifest=self._dbt_manifest, ) @@ -102,7 +98,7 @@ def validate_reference( ref_package: str | None, owner_relative_sql_path: Path, ) -> None: - validate_compile_model_reference( + validate_compile_dbt_model_reference( reference=CompileSqlReference( ref_kind=ref_kind, ref_name=ref_name, diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_build_compile_manifest_index.py b/src/sqlbuild/integrations/dbt/main/manifest/_build_compile_manifest_index.py deleted file mode 100644 index 2779ca44f4..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_build_compile_manifest_index.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public dbt compile manifest index entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.manifest.compile_refs import ( - build_compile_dbt_manifest_index, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def build_compile_manifest_index(*, manifest_contents: str | None) -> DbtManifestIndex | None: - """Build a dbt manifest index from discovered manifest contents.""" - - return build_compile_dbt_manifest_index(manifest_contents=manifest_contents) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_load_manifest_index.py b/src/sqlbuild/integrations/dbt/main/manifest/_load_manifest_index.py deleted file mode 100644 index df2f1281cf..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_load_manifest_index.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public dbt manifest entrypoints.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.integrations.dbt._helpers.manifest.core import ( - load_dbt_manifest_index, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def load_manifest_index(*, manifest_path: Path) -> DbtManifestIndex: - """Load and index a dbt manifest file.""" - - return load_dbt_manifest_index(manifest_path=manifest_path) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_manifest_model_materialization.py b/src/sqlbuild/integrations/dbt/main/manifest/_manifest_model_materialization.py deleted file mode 100644 index 9d744d6292..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_manifest_model_materialization.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Resolve dbt manifest model materialization.""" - -from sqlbuild.integrations.dbt._helpers.manifest.core import ( - dbt_manifest_model_materialization as _resolve, -) -from sqlbuild.integrations.dbt.models import DbtManifestModel - - -def dbt_manifest_model_materialization(*, model: DbtManifestModel) -> str | None: - """Return a manifest model's normalized materialization.""" - - return _resolve(model=model) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_reference_relation.py b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_reference_relation.py deleted file mode 100644 index cc96627b27..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_reference_relation.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Resolve a dbt reference relation.""" - -from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( - resolve_dbt_reference_relation as _resolve, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def resolve_dbt_reference_relation( - *, - manifest: DbtManifestIndex | None, - ref_kind: str, - ref_name: str, - ref_package: str | None, -) -> str | None: - """Resolve one external dbt reference to its relation.""" - - return _resolve( - manifest=manifest, - ref_kind=ref_kind, - ref_name=ref_name, - ref_package=ref_package, - ) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py deleted file mode 100644 index 22d3daf843..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Resolve SQLBuild model references to dbt models.""" - -from collections.abc import Sequence - -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( - resolve_sqlbuild_model_dbt_refs as _resolve, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex, DbtManifestModel - - -def resolve_sqlbuild_model_dbt_refs( - *, - project: CompiledProjectView, - manifest: DbtManifestIndex, - selected_model_names: Sequence[str] | None = None, -) -> tuple[tuple[CompiledModelView, DbtManifestModel], ...]: - """Return dbt refs used directly by SQLBuild models.""" - - return _resolve(project=project, manifest=manifest, selected_model_names=selected_model_names) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_names.py b/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_names.py deleted file mode 100644 index 2a7386338c..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_names.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public dbt compile duplicate model-name validation entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.manifest.compile_refs import ( - validate_compile_dbt_model_names, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def validate_compile_model_names( - *, known_model_names: set[str], dbt_manifest: DbtManifestIndex | None -) -> None: - """Reject dbt/SQLBuild duplicate logical model names.""" - - validate_compile_dbt_model_names( - known_model_names=known_model_names, - dbt_manifest=dbt_manifest, - ) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_reference.py b/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_reference.py deleted file mode 100644 index e17fc11e99..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_reference.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public dbt compile model ref validation entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.integrations.dbt._helpers.manifest.compile_refs import ( - validate_compile_dbt_model_reference, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def validate_compile_model_reference( - *, - reference: CompileSqlReference, - model_relative_path: Path, - dbt_manifest: DbtManifestIndex | None, -) -> None: - """Validate a SQLBuild model __dbt_ref() against a dbt manifest.""" - - validate_compile_dbt_model_reference( - reference=reference, - model_relative_path=model_relative_path, - dbt_manifest=dbt_manifest, - ) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py b/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py index cdac9b4eb6..bb39ded142 100644 --- a/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py +++ b/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py @@ -2,13 +2,11 @@ from __future__ import annotations +from sqlbuild._native import build_compile_dbt_manifest_index from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver import ( DbtCompileReferenceResolver, ) -from sqlbuild.integrations.dbt.main.manifest._build_compile_manifest_index import ( - build_compile_manifest_index, -) def build_compile_reference_resolver( @@ -19,5 +17,5 @@ def build_compile_reference_resolver( if manifest_contents is None: return None return DbtCompileReferenceResolver( - dbt_manifest=build_compile_manifest_index(manifest_contents=manifest_contents) + dbt_manifest=build_compile_dbt_manifest_index(manifest_contents=manifest_contents) ) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index f34f336cba..4a8e449605 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -5,7 +5,12 @@ import pytest -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + build_dbt_manifest_index, + discover_project_inputs, +) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.graph.core import ( @@ -13,7 +18,6 @@ dbt_model_graph_key, expand_combined_downstream, ) -from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import ( DbtCliOptions, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 76115fd470..6fe28897c9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import build_dbt_manifest_index from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -19,7 +20,6 @@ resolve_udf_references, ) from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver import ( DbtCompileReferenceResolver, ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py b/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py index 936f1bdaff..7d37df9880 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py @@ -2,13 +2,13 @@ import pytest +from sqlbuild._native import build_dbt_manifest_index from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.graph.core import ( build_dbt_combined_graph, expand_combined_downstream, expand_combined_upstream, ) -from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt.models import DbtCombinedGraph, DbtManifestIndex from tests.unit.src.sqlbuild.integrations.dbt._test_types import DbtCombinedGraphTestCase from tests.unit.src.sqlbuild.integrations.dbt.helpers import ( diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py index e0bf8805b1..a0f9cafdca 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py @@ -2,11 +2,11 @@ import pytest -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.integrations.dbt._helpers.manifest.core import ( +from sqlbuild._native import ( build_dbt_manifest_index, resolve_dbt_manifest_model, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.integrations.dbt.models import ( DbtManifestIndex, DbtManifestModel, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py index 77e56f82b8..bf0deca744 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py @@ -4,10 +4,10 @@ import pytest +from sqlbuild._native import build_dbt_manifest_index from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_ls_argv from sqlbuild.integrations.dbt._helpers.graph.core import build_dbt_combined_graph -from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt._helpers.planning.orchestration import plan_dbt_interop_command from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py index 2add74343b..b564c604da 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py @@ -2,9 +2,9 @@ import pytest +from sqlbuild._native import build_dbt_manifest_index from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.graph.core import build_dbt_combined_graph -from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt._helpers.selection.core import resolve_dbt_interop_sqlbuild_selection from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError from sqlbuild.integrations.dbt.models import ( From ccb38a669f5654d208e3c4b22b19a9b4c16d6e69 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:54:04 +0100 Subject: [PATCH 09/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 4de5c7383d..9d2b8b7a56 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 88471 +remaining_lines = 87254 [[stays]] path = "executor" From 3001f2bccfda109adfac24a7ff17d624764bdadd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:29:25 +0100 Subject: [PATCH 10/37] refactor(cli): run sqb load natively --- crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- crates/sqlbuild-core/src/lib.rs | 1 + .../src/load_command/_helpers/graph.rs | 116 ++++ .../src/load_command/_helpers/mod.rs | 3 + .../src/load_command/_helpers/padding.rs | 24 + .../load_command/_helpers/selection_sets.rs | 190 ++++++ .../src/load_command/constants.rs | 15 + .../sqlbuild-core/src/load_command/errors.rs | 7 + .../src/load_command/main/load_counts.rs | 24 + .../src/load_command/main/load_exit_code.rs | 9 + .../main/load_reference_loaders.rs | 28 + .../src/load_command/main/mod.rs | 10 + .../load_command/main/render_empty_load.rs | 15 + .../load_command/main/render_load_footer.rs | 60 ++ .../src/load_command/main/render_load_plan.rs | 41 ++ .../load_command/main/render_load_ready.rs | 13 + .../load_command/main/render_load_result.rs | 53 ++ .../main/render_load_spinner_line.rs | 24 + .../load_command/main/select_load_entries.rs | 51 ++ crates/sqlbuild-core/src/load_command/mod.rs | 10 + .../sqlbuild-core/src/load_command/models.rs | 75 +++ .../sqlbuild-core/src/load_command/tests.rs | 6 + .../src/load_command/tests/helpers.rs | 52 ++ .../load_command/tests/test_load_command.rs | 199 ++++++ .../src/load_command/tests/test_types.rs | 28 + .../bindings/_helpers/load/load_command.rs | 604 +++++++++++++++++ .../bindings/_helpers/load/load_progress.rs | 428 ++++++++++++ .../bindings/_helpers/load/load_selection.rs | 400 +++++++++++ .../src/bindings/_helpers/load/mod.rs | 4 + .../bindings/_helpers/load/registration.rs | 11 + .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/main/register.rs | 7 +- src/sqlbuild/_native.pyi | 37 + .../cli/commands/_helpers/load/execution.py | 119 ---- .../cli/commands/_helpers/load/invocation.py | 110 --- .../cli/commands/_helpers/load/outputs.py | 146 ---- .../cli/commands/_helpers/load/planning.py | 112 ---- .../cli/commands/_helpers/load/references.py | 53 -- .../cli/commands/_helpers/load/selection.py | 632 ------------------ .../classes/load_progress_reporter.py | 265 -------- .../cli/commands/main/execution/_load.py | 47 -- src/sqlbuild/cli/commands/models.py | 60 -- .../cli/commands/main/test_load_command.py | 61 +- .../_helpers/load_selection/test_selection.py | 6 +- .../cli/commands/main/entry/helpers.py | 4 +- .../src/sqlbuild/executor/load/test_run.py | 3 +- 46 files changed, 2590 insertions(+), 1578 deletions(-) create mode 100644 crates/sqlbuild-core/src/load_command/_helpers/graph.rs create mode 100644 crates/sqlbuild-core/src/load_command/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/load_command/_helpers/padding.rs create mode 100644 crates/sqlbuild-core/src/load_command/_helpers/selection_sets.rs create mode 100644 crates/sqlbuild-core/src/load_command/constants.rs create mode 100644 crates/sqlbuild-core/src/load_command/errors.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/load_counts.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/load_exit_code.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/load_reference_loaders.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/mod.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/render_empty_load.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/render_load_footer.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/render_load_plan.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/render_load_ready.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/render_load_result.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/render_load_spinner_line.rs create mode 100644 crates/sqlbuild-core/src/load_command/main/select_load_entries.rs create mode 100644 crates/sqlbuild-core/src/load_command/mod.rs create mode 100644 crates/sqlbuild-core/src/load_command/models.rs create mode 100644 crates/sqlbuild-core/src/load_command/tests.rs create mode 100644 crates/sqlbuild-core/src/load_command/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/load_command/tests/test_load_command.rs create mode 100644 crates/sqlbuild-core/src/load_command/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/load/execution.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/load/invocation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/load/outputs.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/load/planning.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/load/references.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/load/selection.py delete mode 100644 src/sqlbuild/cli/commands/classes/load_progress_reporter.py delete mode 100644 src/sqlbuild/cli/commands/main/execution/_load.py diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index e3fe296a2a..e7d4191d10 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -310,9 +310,9 @@ const LOAD_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const LOAD_ROUTE: Route = routed( "run_load", - ("sqlbuild.cli.commands.main.execution._load", "run_load"), + ("sqlbuild._native", "run_load_command"), &LOAD_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const SEED_FIELDS: [Field; 9] = [ diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 1be30d1a0f..196d5a730a 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod freshness_text; pub mod integration_results; pub mod janitor_text; pub mod json; +pub mod load_command; pub mod models; pub mod panics; pub mod posix_path; diff --git a/crates/sqlbuild-core/src/load_command/_helpers/graph.rs b/crates/sqlbuild-core/src/load_command/_helpers/graph.rs new file mode 100644 index 0000000000..2d337e6e96 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/_helpers/graph.rs @@ -0,0 +1,116 @@ +//! Loader graph lookups: dependencies, closures and execution order. + +use std::collections::{BTreeSet, HashMap}; + +use crate::load_command::models::LoadGraph; + +/// Indexed views of a load graph, as the Python dictionaries keyed them. +pub(crate) struct GraphIndex { + /// Managed source name to its loader; a later duplicate wins. + pub(crate) source_loader: HashMap, + /// Managed source names in declaration order, first occurrence only. + pub(crate) source_order: Vec, + /// Loader name to its dependencies; a later duplicate wins. + pub(crate) upstream: HashMap>, + /// Loader name to its sources in declaration order. + pub(crate) sources_by_loader: HashMap>, +} + +impl GraphIndex { + pub(crate) fn new(graph: &LoadGraph) -> Self { + let mut source_loader: HashMap = HashMap::new(); + let mut source_order: Vec = Vec::new(); + for (name, loader) in &graph.sources { + if source_loader.insert(name.clone(), loader.clone()).is_none() { + source_order.push(name.clone()); + } + } + let mut sources_by_loader: HashMap> = HashMap::new(); + for name in &source_order { + sources_by_loader + .entry(source_loader[name].clone()) + .or_default() + .push(name.clone()); + } + Self { + source_loader, + source_order, + upstream: graph.loaders.iter().cloned().collect(), + sources_by_loader, + } + } + + pub(crate) fn is_loader(&self, name: &str) -> bool { + self.upstream.contains_key(name) + } + + pub(crate) fn dependencies(&self, loader: &str) -> &[String] { + self.upstream.get(loader).map_or(&[], Vec::as_slice) + } + + /// The loader and every loader it transitively depends on. + pub(crate) fn upstream_closure(&self, loader: &str) -> BTreeSet { + let mut closure: BTreeSet = BTreeSet::new(); + let mut pending: Vec = vec![loader.to_owned()]; + while let Some(next) = pending.pop() { + if closure.insert(next.clone()) { + pending.extend(self.dependencies(&next).iter().cloned()); + } + } + closure + } + + /// The loader, every loader depending on it transitively, and all their sources. + pub(crate) fn downstream_closure(&self, loader: &str) -> (BTreeSet, BTreeSet) { + let mut loaders: BTreeSet = BTreeSet::new(); + let mut pending: Vec = vec![loader.to_owned()]; + while let Some(next) = pending.pop() { + if !loaders.insert(next.clone()) { + continue; + } + for (candidate, dependencies) in &self.upstream { + if dependencies.contains(&next) { + pending.push(candidate.clone()); + } + } + } + let sources: BTreeSet = loaders + .iter() + .flat_map(|name| { + self.sources_by_loader + .get(name) + .into_iter() + .flatten() + .cloned() + }) + .collect(); + (loaders, sources) + } + + /// `names` ordered dependencies first, visiting in sorted order. + pub(crate) fn topological_order(&self, names: &BTreeSet) -> Vec { + let mut ordered: Vec = Vec::new(); + let mut visited: BTreeSet = BTreeSet::new(); + for name in names { + self.visit(name, names, (&mut ordered, &mut visited)); + } + ordered + } + + fn visit( + &self, + name: &str, + names: &BTreeSet, + (ordered, visited): (&mut Vec, &mut BTreeSet), + ) { + if !visited.insert(name.to_owned()) { + return; + } + for dependency in self.dependencies(name) { + if names.contains(dependency) { + self.visit(dependency, names, (ordered, visited)); + } + } + ordered.push(name.to_owned()); + } +} diff --git a/crates/sqlbuild-core/src/load_command/_helpers/mod.rs b/crates/sqlbuild-core/src/load_command/_helpers/mod.rs new file mode 100644 index 0000000000..06011aeaf4 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod graph; +pub(crate) mod padding; +pub(crate) mod selection_sets; diff --git a/crates/sqlbuild-core/src/load_command/_helpers/padding.rs b/crates/sqlbuild-core/src/load_command/_helpers/padding.rs new file mode 100644 index 0000000000..2af6b2a40f --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/_helpers/padding.rs @@ -0,0 +1,24 @@ +//! Python format-spec padding and digit grouping. + +/// `f"{text: String { + let length: usize = text.chars().count(); + format!("{text}{}", " ".repeat(width.saturating_sub(length))) +} + +/// `f"{value:,}"`: the integer with comma-separated thousands. +pub(crate) fn group_thousands(value: i64) -> String { + let digits: String = value.unsigned_abs().to_string(); + let mut grouped: String = String::new(); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + grouped.push(','); + } + grouped.push(digit); + } + if value < 0 { + format!("-{grouped}") + } else { + grouped + } +} diff --git a/crates/sqlbuild-core/src/load_command/_helpers/selection_sets.rs b/crates/sqlbuild-core/src/load_command/_helpers/selection_sets.rs new file mode 100644 index 0000000000..17635232b1 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/_helpers/selection_sets.rs @@ -0,0 +1,190 @@ +//! Applying load selectors and excludes to the selected source and loader sets. + +use std::collections::BTreeSet; + +use crate::load_command::_helpers::graph::GraphIndex; +use crate::load_command::constants::UPSTREAM_MARK; +use crate::load_command::errors::UnknownLoadSelector; + +/// Selected sources and loaders, and the loaders selected by name. +#[derive(Default)] +pub(crate) struct Selection { + pub(crate) sources: BTreeSet, + pub(crate) loaders: BTreeSet, + pub(crate) direct_loaders: BTreeSet, + pub(crate) excluded_loaders: BTreeSet, +} + +/// The selector name and whether it asks for upstream and downstream nodes. +fn parse(selector: &str) -> (String, bool, bool) { + let stripped: &str = selector.trim(); + ( + stripped.trim_matches(UPSTREAM_MARK).to_owned(), + stripped.starts_with(UPSTREAM_MARK), + stripped.ends_with(UPSTREAM_MARK), + ) +} + +fn validate(index: &GraphIndex, name: &str) -> Result<(), UnknownLoadSelector> { + if index.source_loader.contains_key(name) || index.is_loader(name) { + return Ok(()); + } + Err(UnknownLoadSelector { + name: name.to_owned(), + }) +} + +/// The loader a downstream selector starts from. +fn downstream_root<'a>(index: &'a GraphIndex, name: &'a str) -> &'a str { + if index.is_loader(name) { + return name; + } + &index.source_loader[name] +} + +impl Selection { + fn add_loader(&mut self, index: &GraphIndex, (loader, upstream): (&str, bool)) { + self.loaders.insert(loader.to_owned()); + if upstream { + self.loaders.extend(index.upstream_closure(loader)); + } + } + + fn select_name(&mut self, index: &GraphIndex, (name, upstream): (&str, bool)) { + if let Some(loader) = index.source_loader.get(name) { + self.sources.insert(name.to_owned()); + self.add_loader(index, (loader, upstream)); + } else if let Some(sources) = index.sources_by_loader.get(name) { + self.sources.extend(sources.iter().cloned()); + self.add_loader(index, (name, upstream)); + } else if index.is_loader(name) { + self.direct_loaders.insert(name.to_owned()); + self.add_loader(index, (name, upstream)); + } + } + + /// Apply each selector in order. + pub(crate) fn select( + &mut self, + index: &GraphIndex, + selectors: &[String], + ) -> Result<(), UnknownLoadSelector> { + for selector in selectors { + let (name, upstream, downstream) = parse(selector); + validate(index, &name)?; + self.select_name(index, (&name, upstream)); + if downstream { + let (loaders, sources) = index.downstream_closure(downstream_root(index, &name)); + self.loaders.extend(loaders); + self.sources.extend(sources); + } + } + Ok(()) + } + + fn exclude_name(&mut self, index: &GraphIndex, name: &str) { + if let Some(loader) = index.source_loader.get(name) { + self.sources.remove(name); + let still_used: bool = index + .sources_by_loader + .get(loader) + .into_iter() + .flatten() + .any(|source| self.sources.contains(source)); + if !still_used { + self.loaders.remove(loader); + } + } else if let Some(sources) = index.sources_by_loader.get(name) { + for source in sources { + self.sources.remove(source); + } + self.loaders.remove(name); + self.excluded_loaders.insert(name.to_owned()); + } else if index.is_loader(name) { + self.loaders.remove(name); + self.excluded_loaders.insert(name.to_owned()); + } + } + + /// Apply each exclude in order. + pub(crate) fn exclude( + &mut self, + index: &GraphIndex, + excludes: &[String], + ) -> Result<(), UnknownLoadSelector> { + for selector in excludes { + let (name, _, downstream) = parse(selector); + validate(index, &name)?; + self.exclude_name(index, &name); + if downstream { + let (loaders, sources) = index.downstream_closure(downstream_root(index, &name)); + self.loaders.retain(|loader| !loaders.contains(loader)); + self.sources.retain(|source| !sources.contains(source)); + } + } + Ok(()) + } + + /// Whether a dependency of `loader` is unselected and blocked. + fn has_blocked_dependency( + &self, + index: &GraphIndex, + (loader, blocked): (&str, &BTreeSet), + ) -> bool { + for dependency in index.dependencies(loader) { + if !self.loaders.contains(dependency) && blocked.contains(dependency) { + return true; + } + } + false + } + + /// Drop loaders whose dependency was excluded or dropped, and the sources they run. + pub(crate) fn prune_missing_dependencies(&mut self, index: &GraphIndex) { + let mut blocked: BTreeSet = self.excluded_loaders.clone(); + loop { + let missing: Vec = self + .loaders + .iter() + .filter(|loader| self.has_blocked_dependency(index, (loader, &blocked))) + .cloned() + .collect(); + let orphaned: Vec = self + .sources + .iter() + .filter(|source| !self.loaders.contains(&index.source_loader[*source])) + .cloned() + .collect(); + if missing.is_empty() && orphaned.is_empty() { + return; + } + for loader in missing { + self.loaders.remove(&loader); + blocked.insert(loader); + } + for source in orphaned { + self.sources.remove(&source); + } + } + } + + /// Keep only loaders a selected source or a by-name loader needs. + pub(crate) fn prune_unneeded_loaders(&mut self, index: &GraphIndex) { + let mut needed: BTreeSet = self.direct_loaders.clone(); + for source in &self.sources { + let loader: &String = &index.source_loader[source]; + if self.loaders.contains(loader) { + needed.insert(loader.clone()); + } + } + let mut pending: Vec = needed.iter().cloned().collect(); + while let Some(loader) = pending.pop() { + for dependency in index.dependencies(&loader) { + if self.loaders.contains(dependency) && needed.insert(dependency.clone()) { + pending.push(dependency.clone()); + } + } + } + self.loaders.retain(|loader| needed.contains(loader)); + } +} diff --git a/crates/sqlbuild-core/src/load_command/constants.rs b/crates/sqlbuild-core/src/load_command/constants.rs new file mode 100644 index 0000000000..2641036745 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/constants.rs @@ -0,0 +1,15 @@ +//! Load output labels and widths. + +pub(crate) const KIND_WIDTH: usize = 10; +pub(crate) const NAME_WIDTH: usize = 30; +pub(crate) const STATUS_WIDTH: usize = 6; +pub(crate) const SPINNER_NAME_WIDTH: usize = 48; +pub(crate) const LOG_LABEL: &str = " log "; +pub(crate) const LOG_CONTINUATION: &str = " "; +pub(crate) const UPSTREAM_MARK: char = '+'; +pub(crate) const SPINNER_FRAMES: [&str; 10] = [ + "\u{280b}", "\u{2819}", "\u{2839}", "\u{2838}", "\u{283c}", "\u{2834}", "\u{2826}", "\u{2827}", + "\u{2807}", "\u{280f}", +]; +pub(crate) const TREE_BRANCH_GLYPH: &str = "\u{251c}\u{2500}\u{2500}"; +pub(crate) const TREE_LAST_GLYPH: &str = "\u{2514}\u{2500}\u{2500}"; diff --git a/crates/sqlbuild-core/src/load_command/errors.rs b/crates/sqlbuild-core/src/load_command/errors.rs new file mode 100644 index 0000000000..ba14a13df2 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/errors.rs @@ -0,0 +1,7 @@ +//! Load selection errors. + +/// A `--select` or `--exclude` name that matches no managed source or loader. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UnknownLoadSelector { + pub name: String, +} diff --git a/crates/sqlbuild-core/src/load_command/main/load_counts.rs b/crates/sqlbuild-core/src/load_command/main/load_counts.rs new file mode 100644 index 0000000000..ec21daf7a5 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/load_counts.rs @@ -0,0 +1,24 @@ +//! Result counts of one load run. + +use crate::load_command::models::{LoadCounts, LoadResultText, LoadStatus}; + +fn count(results: &[LoadResultText], predicate: impl Fn(&LoadResultText) -> bool) -> i64 { + i64::try_from(results.iter().filter(|result| predicate(result)).count()).unwrap_or(i64::MAX) +} + +/// Successes without warnings pass; successes with warnings warn; failures and skips count apart. +#[must_use] +pub fn load_counts(results: &[LoadResultText]) -> LoadCounts { + let success = |result: &LoadResultText| result.status == LoadStatus::Success; + LoadCounts { + pass: count(results, |result| { + success(result) && result.warning_messages.is_empty() + }), + warn: count(results, |result| { + success(result) && !result.warning_messages.is_empty() + }), + fail: count(results, |result| result.status == LoadStatus::Failed), + skip: count(results, |result| result.status == LoadStatus::Skipped), + total: count(results, |_| true), + } +} diff --git a/crates/sqlbuild-core/src/load_command/main/load_exit_code.rs b/crates/sqlbuild-core/src/load_command/main/load_exit_code.rs new file mode 100644 index 0000000000..d3d137a59f --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/load_exit_code.rs @@ -0,0 +1,9 @@ +//! The exit code of `sqb load`. + +use crate::load_command::models::LoadCounts; + +/// 1 when any load failed, else 0. +#[must_use] +pub fn load_exit_code(counts: LoadCounts) -> i32 { + i32::from(counts.fail != 0) +} diff --git a/crates/sqlbuild-core/src/load_command/main/load_reference_loaders.rs b/crates/sqlbuild-core/src/load_command/main/load_reference_loaders.rs new file mode 100644 index 0000000000..36d346555e --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/load_reference_loaders.rs @@ -0,0 +1,28 @@ +//! Unselected upstream loaders that selected loads only reference. + +use std::collections::BTreeSet; + +use crate::load_command::_helpers::graph::GraphIndex; +use crate::load_command::models::LoadGraph; + +/// Loaders upstream of the selected entries' loaders, in dependency order, minus selected names. +#[must_use] +pub fn load_reference_loaders( + graph: &LoadGraph, + selected: &[(String, Option)], +) -> Vec { + let index: GraphIndex = GraphIndex::new(graph); + let mut references: BTreeSet = BTreeSet::new(); + for (_, loader) in selected { + let Some(loader) = loader else { continue }; + for dependency in index.dependencies(loader) { + references.extend(index.upstream_closure(dependency)); + } + } + let selected_names: BTreeSet<&String> = selected.iter().map(|(name, _)| name).collect(); + index + .topological_order(&references) + .into_iter() + .filter(|loader| index.is_loader(loader) && !selected_names.contains(loader)) + .collect() +} diff --git a/crates/sqlbuild-core/src/load_command/main/mod.rs b/crates/sqlbuild-core/src/load_command/main/mod.rs new file mode 100644 index 0000000000..0eb59a050d --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/mod.rs @@ -0,0 +1,10 @@ +pub mod load_counts; +pub mod load_exit_code; +pub mod load_reference_loaders; +pub mod render_empty_load; +pub mod render_load_footer; +pub mod render_load_plan; +pub mod render_load_ready; +pub mod render_load_result; +pub mod render_load_spinner_line; +pub mod select_load_entries; diff --git a/crates/sqlbuild-core/src/load_command/main/render_empty_load.rs b/crates/sqlbuild-core/src/load_command/main/render_empty_load.rs new file mode 100644 index 0000000000..a28b2983b4 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_empty_load.rs @@ -0,0 +1,15 @@ +//! The output of a load with nothing selected. + +use crate::cli_text::models::CliStyle; +use crate::load_command::main::render_load_footer::render_load_footer; +use crate::load_command::models::LoadCounts; + +/// A note that no managed sources were selected, then a zero-count completion line. +#[must_use] +pub fn render_empty_load(style: CliStyle) -> String { + format!( + "{}\n{}", + style.muted(" No managed sources selected."), + render_load_footer(&[], (LoadCounts::default(), 0.0), style) + ) +} diff --git a/crates/sqlbuild-core/src/load_command/main/render_load_footer.rs b/crates/sqlbuild-core/src/load_command/main/render_load_footer.rs new file mode 100644 index 0000000000..482373d225 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_load_footer.rs @@ -0,0 +1,60 @@ +//! The final load summary: loader warnings, then the completion line. + +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, CompletionState, SummaryCount}; +use crate::load_command::models::{LoadCounts, LoadResultText}; + +fn summary_count(label: &str, value: i64) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value, + } +} + +/// Warnings by source when any, a blank line, and the completion line with counts and elapsed time. +#[must_use] +pub fn render_load_footer( + results: &[LoadResultText], + (counts, elapsed_seconds): (LoadCounts, f64), + style: CliStyle, +) -> String { + let mut text: String = String::new(); + for result in results + .iter() + .filter(|result| !result.warning_messages.is_empty()) + { + if text.is_empty() { + text.push_str(&format!("\n{}\n\n", style.warning_strong("Warnings:"))); + } + text.push_str(&format!( + " {} (loader {})\n", + result.source_name, result.loader_name + )); + for message in &result.warning_messages { + text.push_str(&format!(" {message}\n")); + } + } + let footer: String = format_summary_footer( + style, + &[ + summary_count("PASS", counts.pass), + summary_count("WARN", counts.warn), + summary_count("FAIL", counts.fail), + summary_count("SKIP", counts.skip), + summary_count("TOTAL", counts.total), + ], + Some(&format!("{elapsed_seconds:.2}s")), + ); + let (state, label) = if counts.fail != 0 { + (CompletionState::Fail, "Completed with errors") + } else if counts.warn != 0 { + (CompletionState::Warn, "Completed with warnings") + } else { + (CompletionState::Ok, "Completed successfully") + }; + format!( + "{text}\n{}\n", + format_completion_line(style, state, label, Some(&footer)) + ) +} diff --git a/crates/sqlbuild-core/src/load_command/main/render_load_plan.rs b/crates/sqlbuild-core/src/load_command/main/render_load_plan.rs new file mode 100644 index 0000000000..9cc14e8f16 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_load_plan.rs @@ -0,0 +1,41 @@ +//! The selected loader and source names before execution. + +use crate::cli_text::models::CliStyle; +use crate::load_command::constants::{TREE_BRANCH_GLYPH, TREE_LAST_GLYPH}; + +fn names_section(title: &str, names: &[&str], style: CliStyle) -> String { + let mut text: String = format!("{}\n", style.section(&format!("{title} ({})", names.len()))); + for (index, name) in names.iter().enumerate() { + text.push_str(&format!( + "{} {name}\n", + style.muted(if index + 1 == names.len() { + TREE_LAST_GLYPH + } else { + TREE_BRANCH_GLYPH + }) + )); + } + text.push('\n'); + text +} + +/// `Loaders (n)` when any intermediate loaders run, then `Sources (n)`, each as a tree. +#[must_use] +pub fn render_load_plan(entries: &[(String, bool)], style: CliStyle) -> String { + let loaders: Vec<&str> = entries + .iter() + .filter(|(_, loader)| *loader) + .map(|(name, _)| name.as_str()) + .collect(); + let sources: Vec<&str> = entries + .iter() + .filter(|(_, loader)| !*loader) + .map(|(name, _)| name.as_str()) + .collect(); + let mut text: String = String::new(); + if !loaders.is_empty() { + text.push_str(&names_section("Loaders", &loaders, style)); + } + text.push_str(&names_section("Sources", &sources, style)); + text +} diff --git a/crates/sqlbuild-core/src/load_command/main/render_load_ready.rs b/crates/sqlbuild-core/src/load_command/main/render_load_ready.rs new file mode 100644 index 0000000000..da6c6b5875 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_load_ready.rs @@ -0,0 +1,13 @@ +//! The header announcing how many loads were selected. + +use crate::cli_text::main::format_surface_header::format_surface_header; +use crate::cli_text::models::CliStyle; + +/// A blank line, `Load ready selected`, and a blank line. +#[must_use] +pub fn render_load_ready(selected: usize, style: CliStyle) -> String { + format!( + "\n{}\n\n", + format_surface_header(style, "Load ready", &format!("{selected} selected")) + ) +} diff --git a/crates/sqlbuild-core/src/load_command/main/render_load_result.rs b/crates/sqlbuild-core/src/load_command/main/render_load_result.rs new file mode 100644 index 0000000000..9d6110524c --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_load_result.rs @@ -0,0 +1,53 @@ +//! The persistent line of one finished load, with its log blocks and error. + +use crate::cli_text::models::CliStyle; +use crate::load_command::_helpers::padding::{group_thousands, pad_right}; +use crate::load_command::constants::{ + KIND_WIDTH, LOG_CONTINUATION, LOG_LABEL, NAME_WIDTH, STATUS_WIDTH, +}; +use crate::load_command::models::{LoadResultText, LoadStatus}; +use crate::text::main::python_splitlines::python_splitlines; + +fn log_block(message: &str, style: CliStyle) -> String { + let lines: Vec<&str> = python_splitlines(message); + let first: &str = lines.first().copied().unwrap_or(""); + let mut text: String = format!("\n{}{}\n", style.label(LOG_LABEL), style.muted(first)); + for line in lines.iter().skip(1) { + text.push_str(&format!( + "{}\n", + style.muted(&format!("{LOG_CONTINUATION}{line}")) + )); + } + text +} + +/// Position, kind, name, status, duration and rows, then log blocks and the error message. +#[must_use] +pub fn render_load_result( + result: &LoadResultText, + (ordinal, total): (i64, i64), + duration_ms: Option, + style: CliStyle, +) -> String { + let status: String = style.status(match result.status { + LoadStatus::Success => "OK", + LoadStatus::Skipped => "SKIP", + LoadStatus::Failed => "FAIL", + }); + let duration: String = + duration_ms.map_or_else(String::new, |ms| format!("{:.2}s", ms / 1000.0)); + let mut text: String = format!( + " {ordinal}/{total} {}{} {} {duration} rows={}\n", + pad_right(&result.resource_kind, KIND_WIDTH), + pad_right(&result.source_name, NAME_WIDTH), + pad_right(&status, STATUS_WIDTH), + group_thousands(result.rows_loaded) + ); + for message in &result.log_messages { + text.push_str(&log_block(message, style)); + } + if let Some(error) = &result.error_message { + text.push_str(&style.error(&format!(" {error}\n"))); + } + text +} diff --git a/crates/sqlbuild-core/src/load_command/main/render_load_spinner_line.rs b/crates/sqlbuild-core/src/load_command/main/render_load_spinner_line.rs new file mode 100644 index 0000000000..dfaa474b1e --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_load_spinner_line.rs @@ -0,0 +1,24 @@ +//! The live row of a running load. + +use crate::cli_text::models::CliStyle; +use crate::load_command::_helpers::padding::pad_right; +use crate::load_command::constants::{KIND_WIDTH, SPINNER_FRAMES, SPINNER_NAME_WIDTH}; +use crate::load_command::models::LoadSpinnerRow; + +/// Carriage return, line clear, and the row with spinner frame `frame`. +#[must_use] +pub fn render_load_spinner_line(row: &LoadSpinnerRow, frame: usize, style: CliStyle) -> String { + let name: String = if row.message.is_empty() { + row.name.clone() + } else { + format!("{} {}", row.name, row.message) + }; + format!( + "\r\u{1b}[K {}/{} {}{} {}", + row.ordinal, + row.total, + pad_right(&row.resource_kind, KIND_WIDTH), + pad_right(&name, SPINNER_NAME_WIDTH), + style.status(SPINNER_FRAMES[frame % SPINNER_FRAMES.len()]) + ) +} diff --git a/crates/sqlbuild-core/src/load_command/main/select_load_entries.rs b/crates/sqlbuild-core/src/load_command/main/select_load_entries.rs new file mode 100644 index 0000000000..69b876aa76 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/select_load_entries.rs @@ -0,0 +1,51 @@ +//! The loaders and sources `sqb load` runs for `--select` and `--exclude`. + +use std::collections::BTreeSet; + +use crate::load_command::_helpers::graph::GraphIndex; +use crate::load_command::_helpers::selection_sets::Selection; +use crate::load_command::constants::UPSTREAM_MARK; +use crate::load_command::errors::UnknownLoadSelector; +use crate::load_command::models::{LoadEntry, LoadGraph}; + +/// Intermediate loaders in dependency order, then selected sources in declaration order. +pub fn select_load_entries( + graph: &LoadGraph, + (select, exclude): (&[String], &[String]), +) -> Result, UnknownLoadSelector> { + let index: GraphIndex = GraphIndex::new(graph); + let selectors: Vec = if select.is_empty() { + index + .source_order + .iter() + .map(|name| format!("{UPSTREAM_MARK}{name}")) + .collect() + } else { + select.to_vec() + }; + let mut selection: Selection = Selection::default(); + selection.select(&index, &selectors)?; + selection.exclude(&index, exclude)?; + selection.prune_missing_dependencies(&index); + selection.prune_unneeded_loaders(&index); + let terminal: BTreeSet<&String> = selection + .sources + .iter() + .map(|source| &index.source_loader[source]) + .collect(); + let mut entries: Vec = index + .topological_order(&selection.loaders) + .into_iter() + .filter(|loader| !terminal.contains(loader)) + .map(LoadEntry::Loader) + .collect(); + entries.extend( + index + .source_order + .iter() + .filter(|name| selection.sources.contains(*name)) + .cloned() + .map(LoadEntry::Source), + ); + Ok(entries) +} diff --git a/crates/sqlbuild-core/src/load_command/mod.rs b/crates/sqlbuild-core/src/load_command/mod.rs new file mode 100644 index 0000000000..adc138fc1c --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/mod.rs @@ -0,0 +1,10 @@ +//! `sqb load`: source and loader selection, progress and summary text, and the exit code. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/load_command/models.rs b/crates/sqlbuild-core/src/load_command/models.rs new file mode 100644 index 0000000000..a0b89719e1 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/models.rs @@ -0,0 +1,75 @@ +//! Load selection graphs, entries and results. + +/// The managed sources and loader functions selection works over. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LoadGraph { + /// Managed sources in declaration order, with the loader each runs. + pub sources: Vec<(String, String)>, + /// Loader functions in discovery order, with the loaders each depends on. + pub loaders: Vec<(String, Vec)>, +} + +/// One entry `sqb load` runs, in execution order. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LoadEntry { + /// An intermediate loader that no selected source runs. + Loader(String), + /// A managed source. + Source(String), +} + +/// How one load ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LoadStatus { + Success, + Skipped, + Failed, +} + +impl LoadStatus { + /// The status from its `ExecutionStatus` value; anything unknown counts as failed. + #[must_use] + pub fn from_value(value: &str) -> Self { + match value { + "success" => Self::Success, + "skipped" => Self::Skipped, + _ => Self::Failed, + } + } +} + +/// One load result as the command renders it. +#[derive(Clone, Debug, PartialEq)] +pub struct LoadResultText { + pub source_name: String, + pub loader_name: String, + /// The `ExecutionResourceKind` value, such as `source` or `loader`. + pub resource_kind: String, + pub status: LoadStatus, + pub rows_loaded: i64, + pub duration_ms: Option, + /// Messages of the result's log lifecycle events. + pub log_messages: Vec, + pub error_message: Option, + pub warning_messages: Vec, +} + +/// Result counts of one load run. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct LoadCounts { + pub pass: i64, + pub warn: i64, + pub fail: i64, + pub skip: i64, + pub total: i64, +} + +/// The source a spinner row shows: position, kind, name and the latest loader message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LoadSpinnerRow { + pub ordinal: i64, + pub total: i64, + pub resource_kind: String, + pub name: String, + pub message: String, +} diff --git a/crates/sqlbuild-core/src/load_command/tests.rs b/crates/sqlbuild-core/src/load_command/tests.rs new file mode 100644 index 0000000000..8c27da4635 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_load_command.rs"] +mod load_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/load_command/tests/helpers.rs b/crates/sqlbuild-core/src/load_command/tests/helpers.rs new file mode 100644 index 0000000000..cca69b32a9 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/tests/helpers.rs @@ -0,0 +1,52 @@ +use crate::load_command::models::{LoadEntry, LoadGraph, LoadResultText, LoadStatus}; + +fn owned(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_owned()).collect() +} + +/// `fetch_events` and `fetch_prices` feed `raw_orders`; `raw_customers` and `raw_regions` stand alone. +pub(super) fn chained_graph() -> LoadGraph { + LoadGraph { + sources: vec![ + ("raw_orders".to_owned(), "raw_orders".to_owned()), + ("raw_customers".to_owned(), "load_customers".to_owned()), + ("raw_regions".to_owned(), "load_customers".to_owned()), + ], + loaders: vec![ + ("fetch_events".to_owned(), Vec::new()), + ("fetch_prices".to_owned(), owned(&["fetch_events"])), + ( + "raw_orders".to_owned(), + owned(&["fetch_events", "fetch_prices"]), + ), + ("load_customers".to_owned(), Vec::new()), + ], + } +} + +pub(super) fn loader(name: &str) -> LoadEntry { + LoadEntry::Loader(name.to_owned()) +} + +pub(super) fn source(name: &str) -> LoadEntry { + LoadEntry::Source(name.to_owned()) +} + +pub(super) fn selectors(names: &[&str]) -> Vec { + owned(names) +} + +/// A finished source load with no logs, warnings or error. +pub(super) fn result(name: &str, status: LoadStatus) -> LoadResultText { + LoadResultText { + source_name: name.to_owned(), + loader_name: name.to_owned(), + resource_kind: "source".to_owned(), + status, + rows_loaded: 0, + duration_ms: None, + log_messages: Vec::new(), + error_message: None, + warning_messages: Vec::new(), + } +} diff --git a/crates/sqlbuild-core/src/load_command/tests/test_load_command.rs b/crates/sqlbuild-core/src/load_command/tests/test_load_command.rs new file mode 100644 index 0000000000..cdb0f54066 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/tests/test_load_command.rs @@ -0,0 +1,199 @@ +use crate::cli_text::models::CliStyle; +use crate::load_command::main::load_counts::load_counts; +use crate::load_command::main::load_exit_code::load_exit_code; +use crate::load_command::main::load_reference_loaders::load_reference_loaders; +use crate::load_command::main::render_load_footer::render_load_footer; +use crate::load_command::main::render_load_result::render_load_result; +use crate::load_command::main::select_load_entries::select_load_entries; +use crate::load_command::models::{LoadResultText, LoadStatus}; +use crate::load_command::tests::helpers::{chained_graph, loader, result, selectors, source}; +use crate::load_command::tests::test_types::{ + FooterTestCase, ReferenceTestCase, ResultLineTestCase, SelectionTestCase, +}; + +#[test] +fn given_selectors_when_selecting_then_orders_loaders_before_sources() { + let test_cases = [ + SelectionTestCase { + description: "no selectors load every source with its upstream loaders", + select: &[], + exclude: &[], + expected_entries: Ok(vec![ + loader("fetch_events"), + loader("fetch_prices"), + source("raw_orders"), + source("raw_customers"), + source("raw_regions"), + ]), + }, + SelectionTestCase { + description: "a bare source runs only its own loader", + select: &["raw_orders"], + exclude: &[], + expected_entries: Ok(vec![source("raw_orders")]), + }, + SelectionTestCase { + description: "a loader name selects all its sources", + select: &["load_customers"], + exclude: &[], + expected_entries: Ok(vec![source("raw_customers"), source("raw_regions")]), + }, + SelectionTestCase { + description: "a trailing plus adds downstream loaders and sources", + select: &[" fetch_prices+ "], + exclude: &[], + expected_entries: Ok(vec![loader("fetch_prices"), source("raw_orders")]), + }, + SelectionTestCase { + description: "excluding a dependency prunes loaders that need it", + select: &[], + exclude: &["fetch_events"], + expected_entries: Ok(vec![source("raw_customers"), source("raw_regions")]), + }, + SelectionTestCase { + description: "excluding one of a loader's sources keeps the loader for the other", + select: &["load_customers"], + exclude: &["raw_regions"], + expected_entries: Ok(vec![source("raw_customers")]), + }, + SelectionTestCase { + description: "an unknown name is rejected", + select: &["+nope"], + exclude: &[], + expected_entries: Err("nope"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + select_load_entries( + &chained_graph(), + (&selectors(test_case.select), &selectors(test_case.exclude)), + ) + .map_err(|error| error.name), + test_case.expected_entries.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selected_sources_when_resolving_references_then_returns_unselected_upstream_loaders() { + let test_cases = [ + ReferenceTestCase { + description: "a terminal source references every upstream loader in order", + selected: vec![("raw_orders".to_owned(), Some("raw_orders".to_owned()))], + expected_loaders: &["fetch_events", "fetch_prices"], + }, + ReferenceTestCase { + description: "selected loaders and sources without loaders are not references", + selected: vec![ + ("fetch_prices".to_owned(), Some("fetch_prices".to_owned())), + ("raw_orders".to_owned(), Some("raw_orders".to_owned())), + ("raw_static".to_owned(), None), + ], + expected_loaders: &["fetch_events"], + }, + ]; + + for test_case in test_cases { + assert_eq!( + load_reference_loaders(&chained_graph(), &test_case.selected), + selectors(test_case.expected_loaders), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_result_when_rendering_then_shows_status_rows_logs_and_error() { + let test_cases = [ + ResultLineTestCase { + description: "a failed loader groups rows and shows its error", + result: LoadResultText { + resource_kind: "loader".to_owned(), + rows_loaded: 1_234_567, + duration_ms: Some(1500.0), + error_message: Some("boom".to_owned()), + ..result("fetch_b", LoadStatus::Failed) + }, + use_color: false, + expected_text: " 2/3 loader fetch_b FAIL 1.50s rows=1,234,567\n boom\n", + }, + ResultLineTestCase { + description: "logs render as dim blocks and colour keeps the status unpadded", + result: LoadResultText { + log_messages: vec!["one\r\ntwo".to_owned()], + ..result("raw_a", LoadStatus::Skipped) + }, + use_color: true, + expected_text: " 2/3 source raw_a \u{1b}[2mSKIP\u{1b}[0m rows=0\n\n\ + \u{1b}[2m log \u{1b}[0m\u{1b}[2mone\u{1b}[0m\n\u{1b}[2m two\u{1b}[0m\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_load_result( + &test_case.result, + (2, 3), + test_case.result.duration_ms, + CliStyle { + use_color: test_case.use_color + }, + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_completing_then_reports_warnings_counts_and_exit_code() { + let test_cases = [ + FooterTestCase { + description: "warnings are listed and make the run complete with warnings", + results: vec![ + LoadResultText { + warning_messages: vec!["[P008] named raw".to_owned()], + loader_name: "load_a".to_owned(), + ..result("raw_a", LoadStatus::Success) + }, + result("raw_b", LoadStatus::Success), + result("raw_c", LoadStatus::Skipped), + ], + expected_text: "\nWarnings:\n\n raw_a (loader load_a)\n [P008] named raw\n\n\ + \u{2713} Completed with warnings PASS=1 WARN=1 FAIL=0 SKIP=1 TOTAL=3 (1.25s)\n", + expected_exit_code: 0, + }, + FooterTestCase { + description: "a failure completes with errors and exits 1", + results: vec![result("raw_a", LoadStatus::Failed)], + expected_text: "\n\u{2717} Completed with errors PASS=0 WARN=0 FAIL=1 SKIP=0 TOTAL=1 (1.25s)\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + let counts = load_counts(&test_case.results); + assert_eq!( + ( + render_load_footer( + &test_case.results, + (counts, 1.25), + CliStyle { use_color: false } + ), + load_exit_code(counts), + ), + ( + test_case.expected_text.to_owned(), + test_case.expected_exit_code + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/load_command/tests/test_types.rs b/crates/sqlbuild-core/src/load_command/tests/test_types.rs new file mode 100644 index 0000000000..31bd8adb72 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/tests/test_types.rs @@ -0,0 +1,28 @@ +use crate::load_command::models::{LoadEntry, LoadResultText}; + +pub(super) struct SelectionTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) exclude: &'static [&'static str], + pub(super) expected_entries: Result, &'static str>, +} + +pub(super) struct ReferenceTestCase { + pub(super) description: &'static str, + pub(super) selected: Vec<(String, Option)>, + pub(super) expected_loaders: &'static [&'static str], +} + +pub(super) struct ResultLineTestCase { + pub(super) description: &'static str, + pub(super) result: LoadResultText, + pub(super) use_color: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct FooterTestCase { + pub(super) description: &'static str, + pub(super) results: Vec, + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs new file mode 100644 index 0000000000..5efc9c3d26 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -0,0 +1,604 @@ +//! `sqb load`: run the selected source loaders and report each load. + +use std::time::Instant; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::load_command::main::load_counts::load_counts; +use sqlbuild_core::load_command::main::load_exit_code::load_exit_code; +use sqlbuild_core::load_command::main::render_empty_load::render_empty_load; +use sqlbuild_core::load_command::main::render_load_footer::render_load_footer; +use sqlbuild_core::load_command::main::render_load_plan::render_load_plan; +use sqlbuild_core::load_command::main::render_load_ready::render_load_ready; +use sqlbuild_core::load_command::models::{LoadCounts, LoadResultText}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::load::load_progress::{ + LoadCallbackRelay, LoadProgressReporter, load_result_text, +}; +use crate::bindings::_helpers::load::load_selection::{ + LoadProject, LoaderDefaults, loader_defaults, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const EFFECTIVE_PACKAGE: &str = "sqlbuild.compiler.compile.main"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; +const CURSOR_OVERRIDES_MODULE: &str = + "sqlbuild.compiler.planner.main.cursor_window.typed_cursor_overrides"; +const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; +const PYTHON_NODE_MODELS: &str = "sqlbuild.python_nodes.models"; +const PYTHON_NODE_TYPES: &str = "sqlbuild.python_nodes.types"; +const CURSOR_PARSING_MODULE: &str = "sqlbuild.cli.commands._helpers.input.parsing"; +const LOAD_MODELS: &str = "sqlbuild.executor.load.models"; +const LOAD_RUN_MODULE: &str = "sqlbuild.executor.load.main.run"; +const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; +const EXECUTION_COMMAND: &str = "sqb load"; + +/// The discovered project, selected loads and output context of one load. +struct Invocation<'py> { + project_dir: Bound<'py, PyAny>, + discovered: Bound<'py, PyAny>, + selected: Vec>, + references: Vec>, + relation_sources: Vec>, + style: CliStyle, + stream: Bound<'py, PyAny>, +} + +/// The adapter, connection, runtime and providers prepared for execution. +struct Preparation<'py> { + adapter_name: String, + adapter: Bound<'py, PyAny>, + connection_config: Bound<'py, PyAny>, + runtime: (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>), + concurrency: i64, + cursor_overrides: Bound<'py, PyAny>, + provider_session: Bound<'py, PyAny>, + project_relations: Bound<'py, PyAny>, +} + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn write(stream: &Bound<'_, PyAny>, value: &str) -> PyResult<()> { + stream.call_method1("write", (value,))?; + stream.call_method0("flush")?; + Ok(()) +} + +fn effective<'py>(py: Python<'py>, (module, name): (&str, &str)) -> PyResult> { + imported(py, &format!("{EFFECTIVE_PACKAGE}.{module}"), name) +} + +fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult> { + let project_dir: Bound<'py, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, + }; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + let select: Bound<'py, PyAny> = request.getattr("select")?; + let exclude: Bound<'py, PyAny> = request.getattr("exclude")?; + call_with( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", select.clone()), + ("exclude", exclude.clone()), + ("discovered_inputs", discovered.clone()), + ], + )?; + let selected_target: Bound<'py, PyAny> = request.getattr("selected_target")?; + let target_config: Bound<'py, PyAny> = call_with( + &effective(py, ("effective_target", "build_effective_target_config"))?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", selected_target.clone()), + ], + )?; + let defaults: LoaderDefaults<'py> = loader_defaults( + &discovered, + ( + &selected_target, + &target_config, + &request.getattr("cli_vars")?, + ), + )?; + let project: LoadProject<'py> = LoadProject::new(&discovered, &target_config)?; + let selected: Vec> = project.selected_entries( + ( + &select.extract::>()?, + &exclude.extract::>()?, + ), + &defaults, + )?; + let references: Vec> = project.reference_entries(&selected, &defaults)?; + let machine_output: bool = request.getattr("json_output")?.extract()?; + let use_color: bool = !request.getattr("no_color")?.extract::()? + && !machine_output + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .extract::()?; + let stream: Bound<'py, PyAny> = + py.import("sys")? + .getattr(if machine_output { "stderr" } else { "stdout" })?; + Ok(Invocation { + project_dir, + discovered, + selected, + references, + relation_sources: project.sources, + style: CliStyle { use_color }, + stream, + }) +} + +fn user_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, ERRORS_MODULE, "CliUserError").and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn target_name<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + let table: Bound<'_, PyAny> = source.getattr("table")?; + if table.is_truthy()? { + Ok(table) + } else { + source.getattr("name") + } +} + +/// Refuse a load whose skipped intermediate loaders have no target relation yet. +fn check_reference_targets( + invocation: &Invocation<'_>, + (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + if invocation.references.is_empty() { + return Ok(()); + } + let py: Python<'_> = adapter.py(); + let selected_names: Vec = invocation + .selected + .iter() + .map(|source| source.getattr("name")?.extract()) + .collect::>()?; + let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (connection_config,))?; + let checked: PyResult<()> = (|| { + let locations: Vec> = invocation + .references + .iter() + .map(|source| { + PyTuple::new( + py, + [ + source.getattr("database")?, + source.getattr("schema")?, + target_name(source)?, + ], + ) + }) + .collect::>()?; + let lookup: Bound<'_, PyAny> = call_with( + &imported(py, RELATION_LOOKUP_MODULE, "build_relation_lookup")?, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("locations", PyTuple::new(py, locations)?.into_any()), + ], + )?; + for source in &invocation.references { + let target: Bound<'_, PyAny> = target_name(source)?; + let exists: bool = call_with( + &lookup.getattr("exists")?, + &[ + ("database", source.getattr("database")?), + ("schema", source.getattr("schema")?), + ("name", target.clone()), + ], + )? + .is_truthy()?; + if !exists { + return Err(user_error( + py, + format!( + "Selected source load requires intermediate loader '{}', but its target \ + relation '{}' does not exist; use +{} to refresh upstream ingress \ + dependencies", + source.getattr("name")?.str()?, + target.str()?, + selected_names.join(", ") + ), + )); + } + } + Ok(()) + })(); + let closed: PyResult> = adapter.call_method1("close", (&connection,)); + checked?; + closed?; + Ok(()) +} + +/// Source relations loaders must not hard-code, or `None` when enforcement is off. +fn project_relations<'py>( + invocation: &Invocation<'py>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + if !invocation + .discovered + .getattr("project_config")? + .getattr("references")? + .getattr("enforce_explicit")? + .is_truthy()? + { + return Ok(py.None().into_bound(py)); + } + let sources: Bound<'py, PyDict> = PyDict::new(py); + for source in invocation + .relation_sources + .iter() + .chain(&invocation.references) + .chain(&invocation.selected) + { + if source.getattr("expression")?.is_none() { + sources.set_item(source.getattr("name")?, source)?; + } + } + let reference_class: Bound<'py, PyAny> = imported(py, PYTHON_NODE_MODELS, "SqlResourceRef")?; + let source_kind: Bound<'py, PyAny> = + imported(py, PYTHON_NODE_TYPES, "SqlResourceRefKind")?.getattr("SOURCE")?; + let render: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "render_source_relation")?; + let relations: Bound<'py, PyDict> = PyDict::new(py); + for (name, source) in sources.iter() { + let key: Bound<'py, PyAny> = call_with( + &reference_class, + &[("kind", source_kind.clone()), ("name", name)], + )?; + let relation: Bound<'py, PyAny> = + call_with(&render, &[("entry", source), ("adapter", adapter.clone())])?; + relations.set_item(key, relation)?; + } + Ok(relations.into_any()) +} + +fn prepare<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: &Bound<'py, PyAny> = &invocation.discovered; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", invocation.project_dir.clone()), + ], + )?; + let connection_config: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, + &[ + ("discovered_inputs", discovered.clone()), + ("project_dir", invocation.project_dir.clone()), + ("selected_target", request.getattr("selected_target")?), + ("cli_vars", request.getattr("cli_vars")?), + ], + )?; + check_reference_targets(invocation, (&adapter, &connection_config))?; + let runtime: (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = call_with( + &effective(py, ("effective_runtime", "build_effective_runtime_config"))?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", request.getattr("selected_target")?), + ("cli_vars", request.getattr("cli_vars")?), + ], + )? + .extract()?; + let requested: Bound<'py, PyAny> = request.getattr("concurrency")?; + let concurrency: i64 = if requested.is_none() { + call_with( + &effective( + py, + ("effective_settings", "build_effective_settings_config"), + )?, + &[("discovered_inputs", discovered.clone())], + )? + .getattr("concurrency")? + .extract()? + } else { + requested.extract()? + }; + let typed: Bound<'py, PyAny> = call_with( + &imported(py, CURSOR_OVERRIDES_MODULE, "typed_cursor_overrides")?, + &[("cursor_overrides", request.getattr("cursor_overrides")?)], + )?; + let cursor_overrides: Bound<'py, PyAny> = if typed.is_truthy()? { + typed + } else { + imported(py, PLANNER_MODELS, "CursorOverrides")?.call0()? + }; + let provider_session: Bound<'py, PyAny> = call_with( + &imported(py, PROVIDER_SESSION_MODULE, "build_provider_session")?, + &[("discovered_providers", discovered.getattr("providers")?)], + )?; + Ok(Preparation { + adapter_name: adapter_name.extract()?, + project_relations: project_relations(invocation, &adapter)?, + adapter, + connection_config, + runtime, + concurrency: concurrency.max(1), + cursor_overrides, + provider_session, + }) +} + +fn cursor<'py>( + overrides: &Bound<'py, PyAny>, + (field, parser): (&str, &str), +) -> PyResult> { + imported(overrides.py(), CURSOR_PARSING_MODULE, parser)?.call1((overrides.getattr(field)?,)) +} + +/// The executor's runtime parameters for this load. +fn runtime_params<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, + preparation: &Preparation<'py>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let (target_name, effective_vars, run_id) = &preparation.runtime; + let overrides: &Bound<'py, PyAny> = &preparation.cursor_overrides; + call_with( + &imported(py, LOAD_MODELS, "LoadRuntimeParams")?, + &[ + ("run_id", run_id.clone()), + ( + "runtime_dir", + invocation + .project_dir + .call_method1("__truediv__", ("target",))?, + ), + ("target", target_name.clone()), + ("vars", effective_vars.clone()), + ("is_reload", request.getattr("reload")?), + ( + "start_cursor_ts", + cursor(overrides, ("start_ts", "parse_cursor_timestamp"))?, + ), + ( + "end_cursor_ts", + cursor(overrides, ("end_ts", "parse_cursor_timestamp"))?, + ), + ( + "start_cursor_int", + cursor(overrides, ("start_int", "parse_cursor_integer"))?, + ), + ( + "end_cursor_int", + cursor(overrides, ("end_int", "parse_cursor_integer"))?, + ), + ( + "use_color", + PyBool::new(py, invocation.style.use_color) + .to_owned() + .into_any(), + ), + ( + "providers", + preparation.provider_session.getattr("providers")?, + ), + ("project_relations", preparation.project_relations.clone()), + ], + ) +} + +/// Run the selected loads with progress callbacks; returns the results and elapsed seconds. +fn execute<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, + preparation: &Preparation<'py>, +) -> PyResult<(Vec>, f64)> { + let py: Python<'py> = request.py(); + let started: Instant = Instant::now(); + let event_writer: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; + let order: Bound<'py, PyDict> = PyDict::new(py); + for (index, source) in invocation.selected.iter().enumerate() { + order.set_item(source.getattr("name")?, index + 1)?; + } + let reporter: Bound<'py, LoadProgressReporter> = + imported(py, NATIVE_MODULE, "LoadProgressReporter")? + .call( + (), + Some(&{ + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("stream", &invocation.stream)?; + options.set_item("use_color", invocation.style.use_color)?; + options.set_item("source_order", order)?; + options.set_item("total_count", invocation.selected.len())?; + options + }), + )? + .downcast_into()?; + let connection: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", text(py, &preparation.adapter_name)), + ("stream", invocation.stream.clone()), + ( + "blank_line_after_complete", + PyBool::new(py, true).to_owned().into_any(), + ), + ( + "use_color", + PyBool::new(py, invocation.style.use_color) + .to_owned() + .into_any(), + ), + ], + )?; + let relay: Bound<'py, LoadCallbackRelay> = Bound::new( + py, + LoadCallbackRelay { + reporter: reporter.clone().unbind(), + event_writer: event_writer.clone().unbind(), + connection: connection.clone().unbind(), + }, + )?; + let executed: PyResult>> = (|| { + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, LOAD_MODELS, "LoadCallbacks")?, + &[ + ("on_load_start", reporter.getattr("on_start")?), + ("on_load_progress", reporter.getattr("on_progress")?), + ("on_load_complete", relay.getattr("on_load_complete")?), + ( + "on_connection_start", + connection.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + relay.getattr("on_connection_complete")?, + ), + ("on_connection_error", relay.getattr("on_connection_error")?), + ], + )?; + call_with( + &imported(py, LOAD_RUN_MODULE, "run_load_pipeline")?, + &[ + ( + "sources", + PyTuple::new(py, &invocation.selected)?.into_any(), + ), + ( + "reference_sources", + PyTuple::new(py, &invocation.references)?.into_any(), + ), + ( + "loader_functions", + invocation.discovered.getattr("loader_functions")?, + ), + ("connection_config", preparation.connection_config.clone()), + ("adapter", preparation.adapter.clone()), + ( + "max_concurrency", + PyInt::new(py, preparation.concurrency).into_any(), + ), + ("runtime", runtime_params(request, invocation, preparation)?), + ("callbacks", callbacks), + ], + )? + .try_iter()? + .collect() + })(); + let closed: PyResult> = event_writer.call_method0("close"); + let results: Vec> = executed?; + closed?; + Ok((results, started.elapsed().as_secs_f64())) +} + +fn write_json(request: &Bound<'_, PyAny>, results: &[Bound<'_, PyAny>]) -> PyResult<()> { + let py: Python<'_> = request.py(); + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_load_execution_json")?, + &[("results", PyTuple::new(py, results)?.into_any())], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(()) +} + +fn loader_node_flag(source: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = source.py(); + Ok(source + .getattr("meta")? + .call_method1("get", (LOADER_NODE_KEY,))? + .is(PyBool::new(py, true).to_owned())) +} + +/// Run one `sqb load` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_load_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let invocation: Invocation<'_> = invocation(py, request)?; + write( + &invocation.stream, + &render_load_ready(invocation.selected.len(), invocation.style), + )?; + if invocation.selected.is_empty() { + write(&invocation.stream, &render_empty_load(invocation.style))?; + write_json(request, &[])?; + return Ok(0); + } + let preparation: Preparation<'_> = prepare(request, &invocation)?; + let outcome: PyResult<(Vec>, f64)> = (|| { + let entries: Vec<(String, bool)> = invocation + .selected + .iter() + .map(|source| { + Ok(( + source.getattr("name")?.extract()?, + loader_node_flag(source)?, + )) + }) + .collect::>()?; + write( + &invocation.stream, + &render_load_plan(&entries, invocation.style), + )?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", &invocation.stream)?; + options.set_item("command", EXECUTION_COMMAND)?; + options.set_item("target", py.None())?; + options.set_item("concurrency", preparation.concurrency)?; + options.set_item("use_color", invocation.style.use_color)?; + imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&options))?; + execute(request, &invocation, &preparation) + })(); + let closed: PyResult> = preparation.provider_session.call_method0("close"); + let (results, elapsed) = outcome?; + closed?; + let texts: Vec = results + .iter() + .map(load_result_text) + .collect::>()?; + let counts: LoadCounts = load_counts(&texts); + write( + &invocation.stream, + &render_load_footer(&texts, (counts, elapsed), invocation.style), + )?; + write_json(request, &results)?; + Ok(load_exit_code(counts)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_load_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs new file mode 100644 index 0000000000..021b9ce147 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs @@ -0,0 +1,428 @@ +//! Live and persistent progress lines of `sqb load`, as executor callbacks. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::load_command::main::render_load_result::render_load_result; +use sqlbuild_core::load_command::main::render_load_spinner_line::render_load_spinner_line; +use sqlbuild_core::load_command::models::{LoadResultText, LoadSpinnerRow, LoadStatus}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const LIFECYCLE_TYPES: &str = "sqlbuild.adapter.contract.types"; +const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; +const SPINNER_TICK_SECONDS: f64 = 0.1; +const SPINNER_JOIN_SECONDS: f64 = 0.2; +const CLEAR_LINE: &str = "\r\u{1b}[K"; +const HIDE_CURSOR: &str = "\u{1b}[?25l"; +const SHOW_CURSOR: &str = "\u{1b}[?25h"; + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +/// One executor `LoadExecutionResult` as the command renders it. +pub(crate) fn load_result_text(result: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = result.py(); + let log_kind: Bound<'_, PyAny> = + imported(py, LIFECYCLE_TYPES, "LifeCycleEventKind")?.getattr("LOG")?; + let mut log_messages: Vec = Vec::new(); + for event in result.getattr("lifecycle_events")?.try_iter()? { + let event: Bound<'_, PyAny> = event?; + if event.getattr("kind")?.eq(&log_kind)? { + log_messages.push(event.getattr("content")?.extract()?); + } + } + let duration: Bound<'_, PyAny> = result.getattr("duration_ms")?; + Ok(LoadResultText { + source_name: result.getattr("source_name")?.extract()?, + loader_name: result.getattr("loader_name")?.str()?.extract()?, + resource_kind: result + .getattr("resource_kind")? + .getattr("value")? + .extract()?, + status: LoadStatus::from_value( + &result + .getattr("status")? + .getattr("value")? + .extract::()?, + ), + rows_loaded: result.getattr("rows_loaded")?.extract()?, + duration_ms: if duration.is_none() { + None + } else { + Some(duration.extract()?) + }, + log_messages, + error_message: optional_text(result.getattr("error_message")?)?, + warning_messages: result.getattr("warning_messages")?.extract()?, + }) +} + +/// The source currently shown on the live row. +#[derive(Clone)] +struct ActiveSource { + name: String, + resource_kind: String, +} + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Default)] +struct ReporterState { + active: Option, + message: String, + frame: usize, + line_active: bool, + cursor_hidden: bool, + stop_event: Option>, + thread: Option>, +} + +/// TTY-aware renderer for `sqb load` execution progress. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct LoadProgressReporter { + stream: Py, + lines: Py, + write_lock: Py, + style: CliStyle, + source_order: HashMap, + total: i64, + is_tty: bool, + projector: Option>, + state: Mutex, +} + +impl LoadProgressReporter { + fn state(&self) -> MutexGuard<'_, ReporterState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn write(&self, py: Python<'_>, text: &str) -> PyResult<()> { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + Ok(()) + } + + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + within(self.write_lock.bind(py), |_| body()) + } + + fn write_spinner_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let (active, message, frame) = { + let mut state = self.state(); + if !state.line_active { + return Ok(()); + } + let Some(active) = state.active.clone() else { + return Ok(()); + }; + let frame: usize = state.frame; + state.frame = (frame + 1) % 10; + (active, state.message.clone(), frame) + }; + let row: LoadSpinnerRow = LoadSpinnerRow { + ordinal: self.source_order[&active.name], + total: self.total, + resource_kind: active.resource_kind, + name: active.name, + message, + }; + self.write(py, &render_load_spinner_line(&row, frame, self.style)) + }) + } + + fn set_cursor(&self, py: Python<'_>, hidden: bool) -> PyResult<()> { + if self.state().cursor_hidden == hidden { + return Ok(()); + } + self.locked(py, || { + self.write(py, if hidden { HIDE_CURSOR } else { SHOW_CURSOR }) + })?; + self.state().cursor_hidden = hidden; + Ok(()) + } + + fn stop_spinner(&self, py: Python<'_>) -> PyResult<()> { + let (event, thread) = { + let mut state = self.state(); + (state.stop_event.take(), state.thread.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + if let Some(thread) = thread { + let thread: &Bound<'_, PyAny> = thread.bind(py); + if thread.call_method0("is_alive")?.is_truthy()? { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("timeout", SPINNER_JOIN_SECONDS)?; + thread.call_method("join", (), Some(&options))?; + } + } + Ok(()) + } + + fn start_spinner(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + slf.get().stop_spinner(py)?; + let event: Bound<'_, PyAny> = imported(py, "threading", "Event")?.call0()?; + let thread: Bound<'_, PyAny> = call_with( + &imported(py, "threading", "Thread")?, + &[ + ("target", slf.getattr("_spin_until_stopped")?), + ("args", PyTuple::new(py, [&event])?.into_any()), + ("daemon", PyBool::new(py, true).to_owned().into_any()), + ], + )?; + { + let mut state = slf.get().state(); + state.stop_event = Some(event.unbind()); + state.thread = Some(thread.clone().unbind()); + } + thread.call_method0("start")?; + Ok(()) + } +} + +#[pymethods] +impl LoadProgressReporter { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "LoadProgressReporter", + options, + (&["stream", "use_color", "source_order", "total_count"], &[]), + )?; + let stream: Bound<'_, PyAny> = arguments.value("stream")?; + let source_order: HashMap = arguments.value("source_order")?.extract()?; + let total_count: i64 = arguments.value("total_count")?.extract()?; + let lines: Bound<'_, PyAny> = + imported(py, LINES_MODULE, "shared_transient_line_coordinator")?.call0()?; + let projector: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + if !projector.is_none() { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("ordinals", source_order.clone())?; + options.set_item("total", total_count)?; + projector.call_method("configure_resources", (), Some(&options))?; + } + Ok(Self { + is_tty: stream.call_method0("isatty")?.is_truthy()?, + stream: stream.unbind(), + write_lock: lines.getattr("lock")?.unbind(), + lines: lines.unbind(), + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + source_order, + total: total_count, + projector: (!projector.is_none()).then(|| projector.unbind()), + state: Mutex::new(ReporterState::default()), + }) + } + + fn on_start(slf: &Bound<'_, Self>, source: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + if reporter.projector.is_some() || !reporter.is_tty { + return Ok(()); + } + let loader_node: bool = source + .getattr("meta")? + .call_method1("get", (LOADER_NODE_KEY,))? + .is(PyBool::new(py, true).to_owned()); + { + let mut state = reporter.state(); + state.active = Some(ActiveSource { + name: source.getattr("name")?.extract()?, + resource_kind: if loader_node { "loader" } else { "source" }.to_owned(), + }); + state.message = String::new(); + } + reporter.set_cursor(py, true)?; + reporter.locked(py, || { + reporter.state().line_active = true; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", reporter.stream.bind(py))?; + options.set_item("owner", slf)?; + reporter + .lines + .bind(py) + .call_method("claim", (), Some(&options))?; + reporter.write_spinner_line(py) + })?; + Self::start_spinner(slf) + } + + fn on_progress( + &self, + py: Python<'_>, + source: &Bound<'_, PyAny>, + message: String, + ) -> PyResult<()> { + if !self.is_tty { + return Ok(()); + } + let name: String = source.getattr("name")?.extract()?; + { + let mut state = self.state(); + match &state.active { + Some(active) if active.name == name => state.message = message, + _ => return Ok(()), + } + } + self.write_spinner_line(py) + } + + fn on_complete(slf: &Bound<'_, Self>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let text: LoadResultText = load_result_text(result)?; + let mut duration_ms: Option = text.duration_ms; + if let Some(projector) = &reporter.projector { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", &text.source_name)?; + options.set_item("resource_id", format!("source:{}", text.source_name))?; + let canonical: Bound<'_, PyAny> = + projector + .bind(py) + .call_method("consume_resource_terminal", (), Some(&options))?; + if canonical.is_none() { + return Ok(()); + } + duration_ms = Some(canonical.extract()?); + } + reporter.stop_spinner(py)?; + if reporter.is_tty { + reporter.locked(py, || { + reporter.state().line_active = false; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("owner", slf)?; + reporter + .lines + .bind(py) + .call_method("release", (), Some(&options))?; + reporter.write(py, CLEAR_LINE) + })?; + reporter.set_cursor(py, false)?; + } + let ordinal: i64 = *reporter + .source_order + .get(&text.source_name) + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(text.source_name.clone()))?; + reporter.write( + py, + &render_load_result( + &text, + (ordinal, reporter.total), + duration_ms, + reporter.style, + ), + ) + } + + /// Erase the live spinner row so a persistent line can take its place. + fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || self.write(py, CLEAR_LINE)) + } + + /// Draw the live spinner row again below persistent output. + fn redraw_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.write_spinner_line(py) + } + + fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.write_spinner_line(py)?; + } + Ok(()) + } +} + +/// Forward load completions to the reporter and event output, and connection timing to its reporter. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct LoadCallbackRelay { + pub(crate) reporter: Py, + pub(crate) event_writer: Py, + pub(crate) connection: Py, +} + +#[pymethods] +impl LoadCallbackRelay { + fn on_load_complete(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + LoadProgressReporter::on_complete(self.reporter.bind(py), result)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("result", result)?; + options.set_item("plan", py.None())?; + options.set_item("command", PyString::new(py, "load"))?; + self.event_writer + .bind(py) + .call_method("write_build_result", (), Some(&options))?; + Ok(()) + } + + fn on_connection_complete( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.connection_call( + py, + "on_connection_complete", + (connection_count, elapsed_seconds), + ) + } + + fn on_connection_error( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.connection_call( + py, + "on_connection_error", + (connection_count, elapsed_seconds), + ) + } +} + +impl LoadCallbackRelay { + fn connection_call( + &self, + py: Python<'_>, + method: &str, + (connection_count, elapsed_seconds): (i64, f64), + ) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("connection_count", connection_count)?; + options.set_item("elapsed_seconds", elapsed_seconds)?; + self.connection + .bind(py) + .call_method(method, (), Some(&options))?; + Ok(()) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs new file mode 100644 index 0000000000..62c40b9cbe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -0,0 +1,400 @@ +//! Resolve which loaders and sources `sqb load` runs, as `SourceEntry` objects. + +use std::collections::HashMap; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::load_command::errors::UnknownLoadSelector; +use sqlbuild_core::load_command::main::load_reference_loaders::load_reference_loaders; +use sqlbuild_core::load_command::main::select_load_entries::select_load_entries as select_entries; +use sqlbuild_core::load_command::models::{LoadEntry, LoadGraph}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; +const DESTINATION_MODULE: &str = "sqlbuild.spec.contracts.main.loader_destination_parts"; +const CONNECTION_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; +const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; +const LOADER_TABLE_PREFIX: &str = "__loader__"; +const UNKNOWN_SELECTOR_CODE: &str = "C901"; +const UNKNOWN_SELECTOR_HELP: &str = "Use exact managed source names or loader function names."; + +/// The default database and schema of intermediate loader tables. +pub(crate) struct LoaderDefaults<'py> { + pub(crate) database: Bound<'py, PyAny>, + pub(crate) schema: Bound<'py, PyAny>, +} + +/// Discovered loader functions by name and the selection graph over managed sources. +pub(crate) struct LoadProject<'py> { + pub(crate) sources: Vec>, + loaders: HashMap>, + graph: LoadGraph, +} + +/// Every declared source with the target's database and schema filled in where unset. +pub(crate) fn environment_sources<'py>( + discovered: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = discovered.py(); + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let mut sources: Vec> = Vec::new(); + for source_file in discovered.getattr("source_files")?.try_iter()? { + for source in source_file?.getattr("source_entries")?.try_iter()? { + let source: Bound<'py, PyAny> = source?; + let changes: Bound<'py, PyDict> = PyDict::new(py); + for field in ["database", "schema"] { + let value: Bound<'py, PyAny> = source.getattr(field)?; + let effective: Bound<'py, PyAny> = if !value.is_none() || target_config.is_none() { + value + } else { + target_config.getattr(field)? + }; + changes.set_item(field, effective)?; + } + sources.push(replace.call((source,), Some(&changes))?); + } + } + Ok(sources) +} + +impl<'py> LoadProject<'py> { + pub(crate) fn new( + discovered: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + ) -> PyResult { + let py: Python<'py> = discovered.py(); + let sources: Vec> = environment_sources(discovered, target_config)?; + let mut graph: LoadGraph = LoadGraph::default(); + for source in &sources { + let loader: Bound<'py, PyAny> = source.getattr("loader")?; + if !loader.is_none() { + graph + .sources + .push((source.getattr("name")?.extract()?, loader.extract()?)); + } + } + let functions: Vec> = discovered + .getattr("loader_functions")? + .try_iter()? + .collect::>()?; + let names_by_function: Bound<'py, PyDict> = PyDict::new(py); + for function in &functions { + names_by_function.set_item(function.getattr("function")?, function.getattr("name")?)?; + } + let mut loaders: HashMap> = HashMap::new(); + for function in functions { + let name: String = function.getattr("name")?.extract()?; + let mut dependencies: Vec = Vec::new(); + for dependency in function.getattr("depends_on")?.try_iter()? { + if let Some(dependency) = names_by_function.get_item(dependency?)? { + dependencies.push(dependency.extract()?); + } + } + graph.loaders.push((name.clone(), dependencies)); + loaders.insert(name, function); + } + Ok(Self { + sources, + loaders, + graph, + }) + } + + /// The `SourceEntry` an intermediate loader writes, under the loader defaults. + fn loader_entry( + &self, + name: &str, + defaults: &LoaderDefaults<'py>, + ) -> PyResult> { + let loader: &Bound<'py, PyAny> = &self.loaders[name]; + let py: Python<'py> = loader.py(); + let destination: Bound<'py, PyAny> = loader.getattr("destination")?; + let destination: Bound<'py, PyAny> = if destination.is_none() { + PyString::new(py, &format!("{LOADER_TABLE_PREFIX}{name}")).into_any() + } else { + destination + }; + let parts: Bound<'py, PyAny> = call_with( + &imported(py, DESTINATION_MODULE, "loader_destination_parts")?, + &[ + ("destination", destination), + ("default_database", defaults.database.clone()), + ("default_schema", defaults.schema.clone()), + ], + )?; + let meta: Bound<'py, PyDict> = PyDict::new(py); + meta.set_item(LOADER_NODE_KEY, true)?; + let name_value: Bound<'py, PyAny> = PyString::new(py, name).into_any(); + call_with( + &imported(py, CONTRACT_MODELS, "SourceEntry")?, + &[ + ("name", name_value.clone()), + ("database", parts.getattr("database")?), + ("schema", parts.getattr("schema")?), + ("table", parts.getattr("table")?), + ("loader", name_value), + ("write_strategy", loader.getattr("write_strategy")?), + ("cursor_column", loader.getattr("cursor_column")?), + ("unique_key", loader.getattr("unique_key")?), + ("contract", loader.getattr("contract")?), + ("meta", meta.into_any()), + ("columns", loader.getattr("columns")?), + ], + ) + } + + /// The selected loader and source entries in execution order. + pub(crate) fn selected_entries( + &self, + (select, exclude): (&[String], &[String]), + defaults: &LoaderDefaults<'py>, + ) -> PyResult>> { + let py: Python<'py> = defaults.database.py(); + let entries: Vec = select_entries(&self.graph, (select, exclude)) + .map_err(|error| unknown_selector(py, &error))?; + let mut managed: HashMap> = HashMap::new(); + for source in &self.sources { + if !source.getattr("loader")?.is_none() { + managed.insert(source.getattr("name")?.extract()?, source); + } + } + entries + .iter() + .map(|entry| match entry { + LoadEntry::Loader(name) => self.loader_entry(name, defaults), + LoadEntry::Source(name) => Ok(managed[name].clone()), + }) + .collect() + } + + /// Unselected upstream loaders the selected entries only reference. + pub(crate) fn reference_entries( + &self, + selected: &[Bound<'py, PyAny>], + defaults: &LoaderDefaults<'py>, + ) -> PyResult>> { + let pairs: Vec<(String, Option)> = selected + .iter() + .map(|source| { + Ok(( + source.getattr("name")?.extract()?, + source.getattr("loader")?.extract()?, + )) + }) + .collect::>()?; + load_reference_loaders(&self.graph, &pairs) + .iter() + .map(|name| self.loader_entry(name, defaults)) + .collect() + } +} + +fn unknown_selector(py: Python<'_>, error: &UnknownLoadSelector) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", UNKNOWN_SELECTOR_CODE)?; + options.set_item("help", UNKNOWN_SELECTOR_HELP)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call( + (format!( + "sqb load selector '{}' does not match any managed source or loader", + error.name + ),), + Some(&options), + ) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `value.strip()` when it is a string with non-blank text, else `None`. +fn non_empty_text<'py>(value: Option>) -> PyResult>> { + let Some(value) = value else { return Ok(None) }; + if !value.is_instance_of::() { + return Ok(None); + } + let stripped: Bound<'py, PyAny> = value.call_method0("strip")?; + Ok(stripped.is_truthy()?.then_some(stripped)) +} + +/// `target_value or fallback`, reading the target field when there is a target. +fn first_truthy<'py>( + target_config: &Bound<'py, PyAny>, + field: &str, + fallback: Option>, +) -> PyResult> { + let py: Python<'py> = target_config.py(); + if !target_config.is_none() { + let value: Bound<'py, PyAny> = target_config.getattr(field)?; + if value.is_truthy()? { + return Ok(value); + } + } + Ok(fallback.unwrap_or_else(|| py.None().into_bound(py))) +} + +/// Intermediate loader defaults: the target namespace, else the connection's (no database on DuckDB). +pub(crate) fn loader_defaults<'py>( + discovered: &Bound<'py, PyAny>, + (selected_target, target_config, cli_vars): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let connection: Bound<'py, PyDict> = call_with( + &imported( + py, + CONNECTION_CONFIG_MODULE, + "build_effective_connection_config", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", selected_target.clone()), + ("cli_vars", cli_vars.clone()), + ], + )? + .downcast_into()?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let duckdb: Bound<'py, PyAny> = + imported(py, ADAPTER_TYPES_MODULE, "BuiltinAdapter")?.getattr("DUCKDB")?; + let connection_database: Option> = if adapter_name.eq(duckdb)? { + None + } else { + non_empty_text(connection.get_item("database")?)? + }; + Ok(LoaderDefaults { + database: first_truthy(target_config, "database", connection_database)?, + schema: first_truthy( + target_config, + "schema", + non_empty_text(connection.get_item("schema")?)?, + )?, + }) +} + +fn names(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Vec::new()); + } + value.extract() +} + +fn passed_defaults<'py>(arguments: &KeywordArguments<'py>) -> PyResult> { + Ok(LoaderDefaults { + database: arguments.value("loader_default_database")?, + schema: arguments.value("loader_default_schema")?, + }) +} + +/// Select source and intermediate loader execution entries for `sqb load`. +#[pyfunction(name = "select_load_entries")] +#[pyo3(signature = (**options))] +fn select_load_entries_entry<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "select_load_entries", + options, + ( + &["discovered_inputs", "select", "exclude", "target_config"], + &["loader_default_database", "loader_default_schema"], + ), + )?; + let project: LoadProject<'py> = LoadProject::new( + &arguments.value("discovered_inputs")?, + &arguments.value("target_config")?, + )?; + let entries: Vec> = project.selected_entries( + ( + &names(&arguments.value("select")?)?, + &names(&arguments.value("exclude")?)?, + ), + &passed_defaults(&arguments)?, + )?; + PyTuple::new(py, entries) +} + +/// Unselected upstream intermediate loader entries the selected entries only reference. +#[pyfunction] +#[pyo3(signature = (**options))] +fn select_load_reference_entries<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "select_load_reference_entries", + options, + ( + &["discovered_inputs", "selected_sources", "target_config"], + &["loader_default_database", "loader_default_schema"], + ), + )?; + let project: LoadProject<'py> = LoadProject::new( + &arguments.value("discovered_inputs")?, + &arguments.value("target_config")?, + )?; + let selected: Vec> = arguments + .value("selected_sources")? + .try_iter()? + .collect::>()?; + PyTuple::new( + py, + project.reference_entries(&selected, &passed_defaults(&arguments)?)?, + ) +} + +/// The default database and schema intermediate loader tables are written to. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_loader_defaults<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_loader_defaults", + options, + ( + &["discovered_inputs"], + &["selected_target", "target_config", "cli_vars"], + ), + )?; + let defaults: LoaderDefaults<'py> = loader_defaults( + &arguments.value("discovered_inputs")?, + ( + &arguments.value("selected_target")?, + &arguments.value("target_config")?, + &arguments.value("cli_vars")?, + ), + )?; + Ok((defaults.database, defaults.schema)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(select_load_entries_entry, module)?)?; + module.add_function(wrap_pyfunction!(select_load_reference_entries, module)?)?; + module.add_function(wrap_pyfunction!(resolve_loader_defaults, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs new file mode 100644 index 0000000000..68a14122c2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod load_command; +pub(crate) mod load_progress; +pub(crate) mod load_selection; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs new file mode 100644 index 0000000000..198a9ed693 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs @@ -0,0 +1,11 @@ +//! Register the native `sqb load` command, its selection entries and progress reporter. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::load::{load_command, load_progress, load_selection}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + load_command::register(module)?; + load_progress::register(module)?; + load_selection::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index e5bc775799..6af3bb6d14 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -27,6 +27,7 @@ pub(crate) mod graph; pub(crate) mod hand_built; pub(crate) mod janitor; pub(crate) mod lineage; +pub(crate) mod load; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod model_config; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 36b87cafca..aa2a3522b2 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -7,9 +7,9 @@ use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, clone, command_runtime, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cost_report, cursor, dag_artifact, dbt_interop, debug_command, - discovery, fingerprints, freshness, frontier, graph, janitor, lineage, macro_bridge, manifest, - model_config, plan_output, presentation_text, project_assembly, refactor, references, render, - result_output, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, + discovery, fingerprints, freshness, frontier, graph, janitor, lineage, load, macro_bridge, + manifest, model_config, plan_output, presentation_text, project_assembly, refactor, references, + render, result_output, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, workspace_commands, }; use crate::bindings::models; @@ -48,6 +48,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { janitor::registration::register(module)?; freshness::registration::register(module)?; clone::registration::register(module)?; + load::registration::register(module)?; plan_output::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 881fbb0072..4c12082f95 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -22,6 +22,7 @@ from sqlbuild.cli.commands.models import ( FreshnessCommandRequest, JanitorCommandRequest, LineageCommandRequest, + LoadCommandRequest, PlaygroundCommandRequest, RefactorCommandRequest, ScopeCommandRequest, @@ -2825,6 +2826,42 @@ def run_refactor_command(*, request: RefactorCommandRequest) -> int: ... def run_janitor_command(*, request: JanitorCommandRequest) -> int: ... def run_freshness_command(*, request: FreshnessCommandRequest) -> int: ... def run_clone_command(*, request: CloneCommandRequest) -> int: ... +def run_load_command(*, request: LoadCommandRequest) -> int: ... +def select_load_entries( + *, + discovered_inputs: DiscoveredProjectInputs, + select: tuple[str, ...], + exclude: tuple[str, ...], + target_config: TargetConfig | None, + loader_default_database: str | None = None, + loader_default_schema: str | None = None, +) -> tuple[SourceEntry, ...]: ... +def select_load_reference_entries( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_sources: tuple[SourceEntry, ...], + target_config: TargetConfig | None, + loader_default_database: str | None = None, + loader_default_schema: str | None = None, +) -> tuple[SourceEntry, ...]: ... +def resolve_loader_defaults( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None = None, + target_config: TargetConfig | None = None, + cli_vars: dict[str, object] | None = None, +) -> tuple[str | None, str | None]: ... + +class LoadProgressReporter: + def __init__( + self, *, stream: TextIO, use_color: bool, source_order: dict[str, int], total_count: int + ) -> None: ... + def on_start(self, source: SourceEntry) -> None: ... + def on_progress(self, source: SourceEntry, message: str) -> None: ... + def on_complete(self, result: LoadExecutionResult) -> None: ... + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + def observe_source_freshness_json( *, adapter: object, diff --git a/src/sqlbuild/cli/commands/_helpers/load/execution.py b/src/sqlbuild/cli/commands/_helpers/load/execution.py deleted file mode 100644 index c3617fe427..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/execution.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Load command execution phase.""" - -from __future__ import annotations - -import time - -from sqlbuild._native import ConnectionProgressReporter, ExecutionEventWriter -from sqlbuild.cli.commands._helpers.input.parsing import ( - parse_cursor_integer, - parse_cursor_timestamp, -) -from sqlbuild.cli.commands.classes.load_progress_reporter import LoadProgressReporter -from sqlbuild.cli.commands.models import ( - LoadCommandRequest, - LoadExecutionPreparation, - LoadInvocation, - LoadRunOutcome, -) -from sqlbuild.executor.load.main.run import run_load_pipeline -from sqlbuild.executor.load.models import ( - LoadCallbacks, - LoadExecutionResult, - LoadRuntimeParams, -) -from sqlbuild.executor.scheduling.types import ExecutionStatus - - -def execute_load_plan( - *, - request: LoadCommandRequest, - invocation: LoadInvocation, - preparation: LoadExecutionPreparation, -) -> LoadRunOutcome: - """Execute selected source loads.""" - - start: float = time.monotonic() - event_writer: ExecutionEventWriter = ExecutionEventWriter() - load_progress: LoadProgressReporter = LoadProgressReporter( - stream=invocation.progress_stream, - use_color=invocation.use_color, - source_order={ - source.name: index for index, source in enumerate(invocation.selected_sources, start=1) - }, - total_count=len(invocation.selected_sources), - ) - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=preparation.adapter_name, - stream=invocation.progress_stream, - blank_line_after_complete=True, - use_color=invocation.use_color, - ) - - def on_load_complete(result: LoadExecutionResult) -> None: - load_progress.on_complete(result) - event_writer.write_build_result(result=result, plan=None, command="load") - - try: - results: tuple[LoadExecutionResult, ...] = run_load_pipeline( - sources=invocation.selected_sources, - reference_sources=invocation.reference_sources, - loader_functions=invocation.discovered_inputs.loader_functions, - connection_config=preparation.connection_config, - adapter=preparation.adapter, - max_concurrency=preparation.effective_concurrency, - runtime=LoadRuntimeParams( - run_id=preparation.run_id, - runtime_dir=invocation.effective_project_dir / "target", - target=preparation.target_name, - vars=preparation.effective_vars, - is_reload=request.reload, - start_cursor_ts=parse_cursor_timestamp( - preparation.effective_cursor_overrides.start_ts - ), - end_cursor_ts=parse_cursor_timestamp(preparation.effective_cursor_overrides.end_ts), - start_cursor_int=parse_cursor_integer( - preparation.effective_cursor_overrides.start_int - ), - end_cursor_int=parse_cursor_integer(preparation.effective_cursor_overrides.end_int), - use_color=invocation.use_color, - providers=preparation.provider_session.providers, - project_relations=preparation.project_relations, - ), - callbacks=LoadCallbacks( - on_load_start=load_progress.on_start, - on_load_progress=lambda source, message: load_progress.on_progress( - source=source, message=message - ), - on_load_complete=on_load_complete, - on_connection_start=connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - ) - finally: - event_writer.close() - return LoadRunOutcome( - results=results, - elapsed=time.monotonic() - start, - success_count=sum( - 1 - for result in results - if result.status == ExecutionStatus.SUCCESS and not result.warning_messages - ), - fail_count=sum(1 for result in results if result.status == ExecutionStatus.FAILED), - skip_count=sum(1 for result in results if result.status == ExecutionStatus.SKIPPED), - warn_count=sum( - 1 - for result in results - if result.status == ExecutionStatus.SUCCESS and result.warning_messages - ), - ) diff --git a/src/sqlbuild/cli/commands/_helpers/load/invocation.py b/src/sqlbuild/cli/commands/_helpers/load/invocation.py deleted file mode 100644 index 0a522f3140..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/invocation.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Load command invocation resolution phase.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import discover_project_inputs -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.cli.commands._helpers.load.selection import ( - select_load_entries, - select_load_reference_entries, - select_load_relation_entries, -) -from sqlbuild.cli.commands.models import LoadCommandRequest, LoadInvocation -from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config -from sqlbuild.compiler.compile.main.effective_target import build_effective_target_config -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import SourceEntry, TargetConfig - - -def resolve_load_invocation(*, request: LoadCommandRequest) -> LoadInvocation: - """Resolve discovery, source selection, and output context for load.""" - - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - reject_unit_test_selectors( - select=request.select, exclude=request.exclude, discovered_inputs=discovered_inputs - ) - target_config: TargetConfig | None = build_effective_target_config( - discovered_inputs=discovered_inputs, - selected_target=request.selected_target, - ) - loader_default_database, loader_default_schema = _effective_loader_defaults( - discovered_inputs=discovered_inputs, - selected_target=request.selected_target, - target_config=target_config, - cli_vars=request.cli_vars, - ) - selected_sources: tuple[SourceEntry, ...] = select_load_entries( - discovered_inputs=discovered_inputs, - select=request.select, - exclude=request.exclude, - target_config=target_config, - loader_default_database=loader_default_database, - loader_default_schema=loader_default_schema, - ) - reference_sources: tuple[SourceEntry, ...] = select_load_reference_entries( - discovered_inputs=discovered_inputs, - selected_sources=selected_sources, - target_config=target_config, - loader_default_database=loader_default_database, - loader_default_schema=loader_default_schema, - ) - machine_output: bool = request.json_output - use_color: bool = not request.no_color and not machine_output and supports_color() - progress_stream: TextIO = sys.stderr if machine_output else sys.stdout - return LoadInvocation( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - selected_sources=selected_sources, - reference_sources=reference_sources, - relation_sources=select_load_relation_entries( - discovered_inputs=discovered_inputs, target_config=target_config - ), - use_color=use_color, - progress_stream=progress_stream, - ) - - -def _effective_loader_defaults( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - target_config: TargetConfig | None, - cli_vars: dict[str, object] | None, -) -> tuple[str | None, str | None]: - connection: dict[str, object] = build_effective_connection_config( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - cli_vars=cli_vars, - ) - connection_database: str | None = _non_empty_string(connection.get("database")) - if ( - resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - == BuiltinAdapter.DUCKDB - ): - connection_database = None - return ( - (target_config.database if target_config is not None else None) or connection_database, - (target_config.schema if target_config is not None else None) - or _non_empty_string(connection.get("schema")), - ) - - -def _non_empty_string(value: object) -> str | None: - return value.strip() if isinstance(value, str) and value.strip() else None diff --git a/src/sqlbuild/cli/commands/_helpers/load/outputs.py b/src/sqlbuild/cli/commands/_helpers/load/outputs.py deleted file mode 100644 index 1f1f2e4247..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/outputs.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Load command output phases.""" - -from __future__ import annotations - -from sqlbuild._native import ( - format_completion_line, - format_load_execution_json, - format_summary_footer, - write_execution_header, - write_execution_json_output, -) -from sqlbuild.cli.commands.classes.load_progress_reporter import format_load_footer -from sqlbuild.cli.commands.models import ( - LoadCommandRequest, - LoadExecutionPreparation, - LoadInvocation, - LoadRunOutcome, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.surface_header import format_surface_header -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.types import CompletionState -from sqlbuild.spec.contracts.models import SourceEntry - - -def write_load_ready_output(*, invocation: LoadInvocation) -> None: - """Write selected source summary header.""" - - style: CliStyle = CliStyle(use_color=invocation.use_color) - ready_header: str = format_surface_header( - style=style, - title="Load ready", - context=f"{len(invocation.selected_sources)} selected", - ) - invocation.progress_stream.write(f"\n{ready_header}\n\n") - invocation.progress_stream.flush() - - -def write_empty_load_output(*, request: LoadCommandRequest, invocation: LoadInvocation) -> None: - """Write successful no-op load output and optional JSON.""" - - style: CliStyle = CliStyle(use_color=invocation.use_color) - invocation.progress_stream.write(style.muted(" No managed sources selected.") + "\n") - counts_summary: str = format_summary_footer( - counts=( - ("PASS", 0), - ("WARN", 0), - ("FAIL", 0), - ("SKIP", 0), - ("TOTAL", 0), - ), - use_color=invocation.use_color, - elapsed="0.00s", - ) - completion: str = format_completion_line( - style=style, - state=CompletionState.OK, - label="Completed successfully", - summary=counts_summary, - ) - invocation.progress_stream.write(f"\n{completion}\n") - invocation.progress_stream.flush() - write_execution_json_output( - payload=format_load_execution_json(results=()), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def write_load_plan_output(*, invocation: LoadInvocation) -> None: - """Write selected loader and source names before execution.""" - - style: CliStyle = CliStyle(use_color=invocation.use_color) - loader_entries: tuple[SourceEntry, ...] = tuple( - source for source in invocation.selected_sources if _is_loader_node(source) - ) - source_entries: tuple[SourceEntry, ...] = tuple( - source for source in invocation.selected_sources if not _is_loader_node(source) - ) - if loader_entries: - invocation.progress_stream.write(style.section(f"Loaders ({len(loader_entries)})")) - invocation.progress_stream.write("\n") - _write_source_names(invocation=invocation, sources=loader_entries) - invocation.progress_stream.write("\n") - invocation.progress_stream.write(style.section(f"Sources ({len(source_entries)})")) - invocation.progress_stream.write("\n") - _write_source_names(invocation=invocation, sources=source_entries) - invocation.progress_stream.write("\n") - invocation.progress_stream.flush() - - -def write_load_execution_header( - *, invocation: LoadInvocation, preparation: LoadExecutionPreparation -) -> None: - """Write direct execution header for load.""" - - write_execution_header( - stream=invocation.progress_stream, - command="sqb load", - target=None, - concurrency=preparation.effective_concurrency, - use_color=invocation.use_color, - ) - - -def write_load_completion_output( - *, request: LoadCommandRequest, invocation: LoadInvocation, outcome: LoadRunOutcome -) -> None: - """Write final load footer and optional JSON.""" - - invocation.progress_stream.write( - format_load_footer( - results=outcome.results, - success_count=outcome.success_count, - warn_count=outcome.warn_count, - fail_count=outcome.fail_count, - skip_count=outcome.skip_count, - elapsed=outcome.elapsed, - use_color=invocation.use_color, - ) - ) - invocation.progress_stream.flush() - write_execution_json_output( - payload=format_load_execution_json(results=outcome.results), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def resolve_load_exit_code(outcome: LoadRunOutcome) -> int: - """Return the shell exit code for a load outcome.""" - - return 0 if outcome.fail_count == 0 else 1 - - -def _write_source_names(*, invocation: LoadInvocation, sources: tuple[SourceEntry, ...]) -> None: - style: CliStyle = CliStyle(use_color=invocation.use_color) - source: SourceEntry - source_index: int - for source_index, source in enumerate(sources): - connector: str = tree_connector(style=style, last=source_index == len(sources) - 1) - invocation.progress_stream.write(f"{connector} {source.name}\n") - - -def _is_loader_node(source: SourceEntry) -> bool: - return source.meta.get("sqlbuild_loader_node") is True diff --git a/src/sqlbuild/cli/commands/_helpers/load/planning.py b/src/sqlbuild/cli/commands/_helpers/load/planning.py deleted file mode 100644 index 98beea16dc..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/planning.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Load command execution preparation phase.""" - -from __future__ import annotations - -from sqlbuild._native import ( - render_source_relation, - resolve_project_connection_config, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.load.references import validate_reference_source_targets -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.models import ( - LoadCommandRequest, - LoadExecutionPreparation, - LoadInvocation, -) -from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config -from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config -from sqlbuild.compiler.planner.main.cursor_window.typed_cursor_overrides import ( - typed_cursor_overrides, -) -from sqlbuild.compiler.planner.models import CursorOverrides -from sqlbuild.provider.main.session import build_provider_session -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import SourceEntry - - -def prepare_load_execution( - *, request: LoadCommandRequest, invocation: LoadInvocation -) -> LoadExecutionPreparation: - """Prepare adapter, connection, runtime, concurrency, and providers for load.""" - - adapter_name: str = resolve_effective_adapter_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, - project_dir=invocation.effective_project_dir, - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=invocation.discovered_inputs, - project_dir=invocation.effective_project_dir, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - validate_reference_source_targets( - adapter=adapter, - connection_config=connection_config, - selected_sources=invocation.selected_sources, - reference_sources=invocation.reference_sources, - ) - target_name: str | None - effective_vars: dict[str, object] - run_id: str - target_name, effective_vars, run_id = build_effective_runtime_config( - discovered_inputs=invocation.discovered_inputs, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - effective_concurrency: int = max( - 1, - request.concurrency - if request.concurrency is not None - else build_effective_settings_config( - discovered_inputs=invocation.discovered_inputs - ).concurrency, - ) - return LoadExecutionPreparation( - adapter_name=adapter_name, - adapter=adapter, - connection_config=connection_config, - target_name=target_name, - effective_vars=effective_vars, - run_id=run_id, - effective_cursor_overrides=( - typed_cursor_overrides(cursor_overrides=request.cursor_overrides) or CursorOverrides() - ), - effective_concurrency=effective_concurrency, - provider_session=build_provider_session( - discovered_providers=invocation.discovered_inputs.providers - ), - project_relations=_load_project_relations(adapter=adapter, invocation=invocation), - ) - - -def _load_project_relations( - *, adapter: BaseAdapter, invocation: LoadInvocation -) -> dict[SqlResourceRef, str] | None: - """Return the source relations loaders must not hard-code, or None when enforcement is off.""" - - if not invocation.discovered_inputs.project_config.references.enforce_explicit: - return None - sources: dict[str, SourceEntry] = { - source.name: source - for source in ( - *invocation.relation_sources, - *invocation.reference_sources, - *invocation.selected_sources, - ) - if source.expression is None - } - return { - SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=name): render_source_relation( - entry=source, adapter=adapter - ) - for name, source in sources.items() - } diff --git a/src/sqlbuild/cli/commands/_helpers/load/references.py b/src/sqlbuild/cli/commands/_helpers/load/references.py deleted file mode 100644 index 067d27f96b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/references.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Helpers for source loader reference-only dependencies.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.spec.contracts.models import SourceEntry - - -def validate_reference_source_targets( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - selected_sources: tuple[SourceEntry, ...], - reference_sources: tuple[SourceEntry, ...], -) -> None: - """Validate skipped intermediate loader reference targets already exist.""" - - if not reference_sources: - return - selected_source_names: str = ", ".join(source.name for source in selected_sources) - connection: object = adapter.connect(connection_config) - try: - relation_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple( - ( - reference_source.database, - reference_source.schema, - reference_source.table or reference_source.name, - ) - for reference_source in reference_sources - ), - ) - reference_source: SourceEntry - for reference_source in reference_sources: - target_name: str = reference_source.table or reference_source.name - if relation_lookup.exists( - database=reference_source.database, - schema=reference_source.schema, - name=target_name, - ): - continue - raise CliUserError( - f"Selected source load requires intermediate loader '{reference_source.name}', " - f"but its target relation '{target_name}' does not exist; use " - f"+{selected_source_names} to refresh upstream ingress dependencies" - ) - finally: - adapter.close(connection) diff --git a/src/sqlbuild/cli/commands/_helpers/load/selection.py b/src/sqlbuild/cli/commands/_helpers/load/selection.py deleted file mode 100644 index 186d5ea33a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/selection.py +++ /dev/null @@ -1,632 +0,0 @@ -"""Selection helpers for sqb load source/loader graphs.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild._native import transitive_closure -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import LoadSelectionSets, LoadSelectorSets -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts -from sqlbuild.spec.contracts.models import LoaderDestinationParts, SourceEntry, TargetConfig - - -def select_load_entries( - *, - discovered_inputs: DiscoveredProjectInputs, - select: tuple[str, ...], - exclude: tuple[str, ...], - target_config: TargetConfig | None, - loader_default_database: str | None = None, - loader_default_schema: str | None = None, -) -> tuple[SourceEntry, ...]: - """Select source and intermediate loader execution entries for sqb load.""" - - discovered_sources: list[SourceEntry] = [] - for source_file in discovered_inputs.source_files: - discovered_sources.extend(source_file.source_entries) - sources: tuple[SourceEntry, ...] = _environment_sources( - sources=tuple(discovered_sources), target_config=target_config - ) - managed_sources: dict[str, SourceEntry] = { - source.name: source for source in sources if source.loader is not None - } - loaders: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in discovered_inputs.loader_functions - } - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in discovered_inputs.loader_functions - } - upstream_loaders: dict[str, tuple[str, ...]] = {} - for loader in discovered_inputs.loader_functions: - dependency_names: list[str] = [] - for dependency in loader.depends_on: - if dependency in loader_name_by_function: - dependency_names.append(loader_name_by_function[dependency]) - upstream_loaders[loader.name] = tuple(dependency_names) - source_by_loader: dict[str, tuple[str, ...]] = _source_names_by_loader(managed_sources) - - selected_sources: set[str] = set() - selected_loaders: set[str] = set() - directly_selected_loaders: set[str] = set() - raw_selectors: tuple[str, ...] = select or tuple(f"+{name}" for name in managed_sources) - selector_sets: LoadSelectorSets = _apply_selectors( - raw_selectors=raw_selectors, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - directly_selected_loaders=directly_selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - upstream_loaders=upstream_loaders, - ) - selected_sources = selector_sets.selected_sources - selected_loaders = selector_sets.selected_loaders - directly_selected_loaders = selector_sets.directly_selected_loaders - exclusion_sets, excluded_loaders = _apply_excludes( - exclude=exclude, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - upstream_loaders=upstream_loaders, - ) - selected_sources = exclusion_sets.selected_sources - selected_loaders = exclusion_sets.selected_loaders - - selected_loaders, selected_sources = _prune_missing_dependencies( - selected_loaders=selected_loaders, - selected_sources=selected_sources, - managed_sources=managed_sources, - upstream_loaders=upstream_loaders, - excluded_loaders=excluded_loaders, - ) - selected_loaders = _prune_unneeded_loaders( - selected_loaders=selected_loaders, - selected_sources=selected_sources, - directly_selected_loaders=directly_selected_loaders, - managed_sources=managed_sources, - upstream_loaders=upstream_loaders, - ) - - entries: list[SourceEntry] = [] - selected_terminal_loaders: set[str | None] = { - managed_sources[source_name].loader for source_name in selected_sources - } - loader_name: str - for loader_name in _topological_loader_order( - loader_names=selected_loaders, - upstream_loaders=upstream_loaders, - ): - if loader_name not in selected_terminal_loaders: - entries.append( - _loader_to_source_entry( - loader=loaders[loader_name], - default_database=loader_default_database, - default_schema=loader_default_schema, - ) - ) - entries.extend( - managed_sources[source_name] - for source_name in sources_order(sources=sources, selected_sources=selected_sources) - ) - return tuple(entries) - - -def select_load_reference_entries( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_sources: tuple[SourceEntry, ...], - target_config: TargetConfig | None, - loader_default_database: str | None = None, - loader_default_schema: str | None = None, -) -> tuple[SourceEntry, ...]: - """Return unselected upstream intermediate loader entries used only for refs.""" - - selected_names: frozenset[str] = frozenset(source.name for source in selected_sources) - loaders: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in discovered_inputs.loader_functions - } - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in discovered_inputs.loader_functions - } - upstream_loaders: dict[str, tuple[str, ...]] = {} - for loader in discovered_inputs.loader_functions: - dependency_names: list[str] = [] - for dependency in loader.depends_on: - if dependency in loader_name_by_function: - dependency_names.append(loader_name_by_function[dependency]) - upstream_loaders[loader.name] = tuple(dependency_names) - reference_loader_names: set[str] = set() - source: SourceEntry - for source in selected_sources: - if source.loader is None or source.loader not in upstream_loaders: - continue - dependency_name: str - for dependency_name in upstream_loaders[source.loader]: - reference_loader_names.update( - _upstream_loader_closure( - loader_name=dependency_name, upstream_loaders=upstream_loaders - ) - ) - return tuple( - _loader_to_source_entry( - loader=loaders[loader_name], - default_database=loader_default_database, - default_schema=loader_default_schema, - ) - for loader_name in _topological_loader_order( - loader_names=reference_loader_names, - upstream_loaders=upstream_loaders, - ) - if loader_name in loaders and loader_name not in selected_names - ) - - -def select_load_relation_entries( - *, discovered_inputs: DiscoveredProjectInputs, target_config: TargetConfig | None -) -> tuple[SourceEntry, ...]: - """Return every declared source resolved for the target, for hard-coded name warnings.""" - - discovered_sources: list[SourceEntry] = [] - for source_file in discovered_inputs.source_files: - discovered_sources.extend(source_file.source_entries) - return _environment_sources(sources=tuple(discovered_sources), target_config=target_config) - - -def sources_order( - *, sources: tuple[SourceEntry, ...], selected_sources: set[str] -) -> tuple[str, ...]: - return tuple(source.name for source in sources if source.name in selected_sources) - - -def _source_names_by_loader(managed_sources: dict[str, SourceEntry]) -> dict[str, tuple[str, ...]]: - grouped_source_names: dict[str, list[str]] = {} - source: SourceEntry - for source in managed_sources.values(): - if source.loader is None: - continue - grouped_source_names.setdefault(source.loader, []).append(source.name) - return { - loader_name: tuple(source_names) - for loader_name, source_names in grouped_source_names.items() - } - - -def _apply_selectors( - *, - raw_selectors: tuple[str, ...], - selected_sources: set[str], - selected_loaders: set[str], - directly_selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - upstream_loaders: dict[str, tuple[str, ...]], -) -> LoadSelectorSets: - selector: str - for selector in raw_selectors: - name, include_upstream, include_downstream = _parse_load_selector(selector) - _validate_load_selector(name=name, managed_sources=managed_sources, loaders=loaders) - selector_sets: LoadSelectorSets = _select_name( - name=name, - include_upstream=include_upstream, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - directly_selected_loaders=directly_selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - upstream_loaders=upstream_loaders, - ) - selected_sources = selector_sets.selected_sources - selected_loaders = selector_sets.selected_loaders - directly_selected_loaders = selector_sets.directly_selected_loaders - if include_downstream: - selection_sets: LoadSelectionSets = _select_downstream( - name=name, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - upstream_loaders=upstream_loaders, - ) - selected_sources = selection_sets.selected_sources - selected_loaders = selection_sets.selected_loaders - return LoadSelectorSets(selected_sources, selected_loaders, directly_selected_loaders) - - -def _apply_excludes( - *, - exclude: tuple[str, ...], - selected_sources: set[str], - selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - upstream_loaders: dict[str, tuple[str, ...]], -) -> tuple[LoadSelectionSets, set[str]]: - excluded_loaders: set[str] = set() - selector: str - for selector in exclude: - name, _, include_downstream = _parse_load_selector(selector) - _validate_load_selector(name=name, managed_sources=managed_sources, loaders=loaders) - selection_sets: LoadSelectionSets - selection_sets, excluded_loaders = _exclude_name( - name=name, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - excluded_loaders=excluded_loaders, - ) - selected_sources = selection_sets.selected_sources - selected_loaders = selection_sets.selected_loaders - if include_downstream: - selection_sets = _exclude_downstream( - name=name, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - upstream_loaders=upstream_loaders, - ) - selected_sources = selection_sets.selected_sources - selected_loaders = selection_sets.selected_loaders - return LoadSelectionSets(selected_sources, selected_loaders), excluded_loaders - - -def _select_name( - *, - name: str, - include_upstream: bool, - selected_sources: set[str], - selected_loaders: set[str], - directly_selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - upstream_loaders: dict[str, tuple[str, ...]], -) -> LoadSelectorSets: - if name in managed_sources: - selected_sources.add(name) - loader_name: str | None = managed_sources[name].loader - if loader_name is not None: - selected_loaders.add(loader_name) - if include_upstream: - selected_loaders.update( - _upstream_loader_closure( - loader_name=loader_name, upstream_loaders=upstream_loaders - ) - ) - return LoadSelectorSets(selected_sources, selected_loaders, directly_selected_loaders) - if name in source_by_loader: - selected_sources.update(source_by_loader[name]) - selected_loaders.add(name) - if include_upstream: - selected_loaders.update( - _upstream_loader_closure(loader_name=name, upstream_loaders=upstream_loaders) - ) - return LoadSelectorSets(selected_sources, selected_loaders, directly_selected_loaders) - if name in loaders: - selected_loaders.add(name) - directly_selected_loaders.add(name) - if include_upstream: - selected_loaders.update( - _upstream_loader_closure(loader_name=name, upstream_loaders=upstream_loaders) - ) - return LoadSelectorSets(selected_sources, selected_loaders, directly_selected_loaders) - - -def _select_downstream( - *, - name: str, - selected_sources: set[str], - selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - upstream_loaders: dict[str, tuple[str, ...]], -) -> LoadSelectionSets: - downstream_loaders, downstream_sources = _downstream_closure( - loader_name=_selected_loader_name( - name=name, - managed_sources=managed_sources, - loaders=loaders, - ), - upstream_loaders=upstream_loaders, - source_by_loader=source_by_loader, - ) - selected_loaders.update(downstream_loaders) - selected_sources.update(downstream_sources) - return LoadSelectionSets(selected_sources, selected_loaders) - - -def _exclude_name( - *, - name: str, - selected_sources: set[str], - selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - excluded_loaders: set[str], -) -> tuple[LoadSelectionSets, set[str]]: - if name in managed_sources: - selected_sources.discard(name) - loader_name: str | None = managed_sources[name].loader - if loader_name is not None and not any( - source_name in selected_sources for source_name in source_by_loader.get(loader_name, ()) - ): - selected_loaders.discard(loader_name) - return LoadSelectionSets(selected_sources, selected_loaders), excluded_loaders - if name in source_by_loader: - source_name: str - for source_name in source_by_loader[name]: - selected_sources.discard(source_name) - selected_loaders.discard(name) - excluded_loaders.add(name) - return LoadSelectionSets(selected_sources, selected_loaders), excluded_loaders - if name in loaders: - selected_loaders.discard(name) - excluded_loaders.add(name) - return LoadSelectionSets(selected_sources, selected_loaders), excluded_loaders - - -def _exclude_downstream( - *, - name: str, - selected_sources: set[str], - selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - upstream_loaders: dict[str, tuple[str, ...]], -) -> LoadSelectionSets: - downstream_loaders, downstream_sources = _downstream_closure( - loader_name=_selected_loader_name( - name=name, - managed_sources=managed_sources, - loaders=loaders, - ), - upstream_loaders=upstream_loaders, - source_by_loader=source_by_loader, - ) - selected_loaders.difference_update(downstream_loaders) - selected_sources.difference_update(downstream_sources) - return LoadSelectionSets(selected_sources, selected_loaders) - - -def _selected_loader_name( - *, - name: str, - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], -) -> str | None: - if name in loaders: - return name - return managed_sources[name].loader - - -def _environment_sources( - *, sources: tuple[SourceEntry, ...], target_config: TargetConfig | None -) -> tuple[SourceEntry, ...]: - return tuple( - replace( - source, - database=source.database - if source.database is not None or target_config is None - else target_config.database, - schema=source.schema - if source.schema is not None or target_config is None - else target_config.schema, - ) - for source in sources - ) - - -def _loader_to_source_entry( - *, - loader: DiscoveredLoaderFunction, - default_database: str | None, - default_schema: str | None, -) -> SourceEntry: - destination: LoaderDestinationParts = loader_destination_parts( - destination=( - loader.destination if loader.destination is not None else f"__loader__{loader.name}" - ), - default_database=default_database, - default_schema=default_schema, - ) - return SourceEntry( - name=loader.name, - database=destination.database, - schema=destination.schema, - table=destination.table, - loader=loader.name, - write_strategy=loader.write_strategy, - cursor_column=loader.cursor_column, - unique_key=loader.unique_key, - contract=loader.contract, - meta={"sqlbuild_loader_node": True}, - columns=loader.columns, - ) - - -def _parse_load_selector(selector: str) -> tuple[str, bool, bool]: - stripped: str = selector.strip() - include_upstream: bool = stripped.startswith("+") - include_downstream: bool = stripped.endswith("+") - name: str = stripped.strip("+") - return name, include_upstream, include_downstream - - -def _validate_load_selector( - *, - name: str, - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], -) -> None: - if name in managed_sources or name in loaders: - return - raise CliUserError( - f"sqb load selector '{name}' does not match any managed source or loader", - code="C901", - help="Use exact managed source names or loader function names.", - ) - - -def _upstream_loader_closure( - *, - loader_name: str | None, - upstream_loaders: dict[str, tuple[str, ...]], -) -> set[str]: - if loader_name is None: - return set() - return {loader_name, *transitive_closure(start=loader_name, edges=upstream_loaders)} - - -def _downstream_closure( - *, - loader_name: str | None, - upstream_loaders: dict[str, tuple[str, ...]], - source_by_loader: dict[str, tuple[str, ...]], -) -> tuple[set[str], set[str]]: - if loader_name is None: - return set(), set() - loaders: set[str] = {loader_name} - sources: set[str] = set(source_by_loader.get(loader_name, ())) - downstream_loader: str - deps: tuple[str, ...] - for downstream_loader, deps in upstream_loaders.items(): - if loader_name not in deps: - continue - next_loaders, next_sources = _downstream_closure( - loader_name=downstream_loader, - upstream_loaders=upstream_loaders, - source_by_loader=source_by_loader, - ) - loaders.update(next_loaders) - sources.update(next_sources) - return loaders, sources - - -def _prune_missing_dependencies( - *, - selected_loaders: set[str], - selected_sources: set[str], - managed_sources: dict[str, SourceEntry], - upstream_loaders: dict[str, tuple[str, ...]], - excluded_loaders: set[str], -) -> tuple[set[str], set[str]]: - pruned_loaders: set[str] = set(selected_loaders) - pruned_sources: set[str] = set(selected_sources) - blocked_loaders: set[str] = set(excluded_loaders) - changed: bool = True - while changed: - changed = False - loader_name: str - for loader_name in tuple(pruned_loaders): - if _has_missing_loader_dependency( - loader_name=loader_name, - selected_loaders=pruned_loaders, - upstream_loaders=upstream_loaders, - blocked_loaders=blocked_loaders, - ): - pruned_loaders.remove(loader_name) - blocked_loaders.add(loader_name) - changed = True - source_name: str - for source_name in tuple(pruned_sources): - source_loader: str | None = managed_sources[source_name].loader - if source_loader is not None and source_loader not in pruned_loaders: - pruned_sources.remove(source_name) - changed = True - return pruned_loaders, pruned_sources - - -def _has_missing_loader_dependency( - *, - loader_name: str, - selected_loaders: set[str], - upstream_loaders: dict[str, tuple[str, ...]], - blocked_loaders: set[str], -) -> bool: - dependencies: tuple[str, ...] = upstream_loaders[loader_name] - return any( - dependency not in selected_loaders and dependency in blocked_loaders - for dependency in dependencies - ) - - -def _prune_unneeded_loaders( - *, - selected_loaders: set[str], - selected_sources: set[str], - directly_selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - upstream_loaders: dict[str, tuple[str, ...]], -) -> set[str]: - needed_loaders: set[str] = set(directly_selected_loaders) - source_name: str - for source_name in selected_sources: - source_loader: str | None = managed_sources[source_name].loader - if source_loader is not None and source_loader in selected_loaders: - needed_loaders.add(source_loader) - - changed: bool = True - while changed: - changed = False - loader_name: str - for loader_name in tuple(needed_loaders): - dependency_name: str - for dependency_name in upstream_loaders.get(loader_name, ()): - if dependency_name in selected_loaders and dependency_name not in needed_loaders: - needed_loaders.add(dependency_name) - changed = True - - return selected_loaders.intersection(needed_loaders) - - -def _topological_loader_order( - *, loader_names: set[str], upstream_loaders: dict[str, tuple[str, ...]] -) -> tuple[str, ...]: - ordered: tuple[str, ...] = () - visited: frozenset[str] = frozenset() - for loader_name in sorted(loader_names): - ordered, visited = _visit_loader( - loader_name=loader_name, - loader_names=loader_names, - upstream_loaders=upstream_loaders, - ordered=ordered, - visited=visited, - ) - return ordered - - -def _visit_loader( - *, - loader_name: str, - loader_names: set[str], - upstream_loaders: dict[str, tuple[str, ...]], - ordered: tuple[str, ...], - visited: frozenset[str], -) -> tuple[tuple[str, ...], frozenset[str]]: - if loader_name in visited: - return ordered, visited - dependency: str - for dependency in upstream_loaders.get(loader_name, ()): - if dependency in loader_names: - ordered, visited = _visit_loader( - loader_name=dependency, - loader_names=loader_names, - upstream_loaders=upstream_loaders, - ordered=ordered, - visited=visited, - ) - return (*ordered, loader_name), visited | {loader_name} diff --git a/src/sqlbuild/cli/commands/classes/load_progress_reporter.py b/src/sqlbuild/cli/commands/classes/load_progress_reporter.py deleted file mode 100644 index b3ed384d04..0000000000 --- a/src/sqlbuild/cli/commands/classes/load_progress_reporter.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Progress callbacks for sqb load execution.""" - -from __future__ import annotations - -import threading -from typing import TextIO - -from sqlbuild._native import ( - NativeProgressProjector, - current_native_progress_projector, - format_completion_line, - format_summary_footer, -) -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator -from sqlbuild.presentation.types import CompletionState -from sqlbuild.spec.contracts.models import SourceEntry - -_SPINNER_TICK_SECONDS: float = 0.1 -_ACTIVE_SPINNER_FRAMES: tuple[str, ...] = ( - "⠋", - "⠙", - "⠹", - "⠸", - "⠼", - "⠴", - "⠦", - "⠧", - "⠇", - "⠏", -) - - -class LoadProgressReporter: - """TTY-aware renderer for sqb load execution progress.""" - - def __init__( - self, - *, - stream: TextIO, - use_color: bool, - source_order: dict[str, int], - total_count: int, - ) -> None: - self._stream: TextIO = stream - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._style: CliStyle = CliStyle(use_color=use_color) - self._source_order: dict[str, int] = source_order - self._total_count: int = total_count - self._is_tty: bool = stream.isatty() - self._current_source: SourceEntry | None = None - self._current_sub_message: str = "" - self._spinner_frame_index: int = 0 - self._spinner_stop_event: threading.Event | None = None - self._spinner_thread: threading.Thread | None = None - self._write_lock: threading.RLock = self._lines.lock - self._spinner_line_active: bool = False - self._cursor_hidden: bool = False - self._projector: NativeProgressProjector | None = current_native_progress_projector() - if self._projector is not None: - self._projector.configure_resources(ordinals=source_order, total=total_count) - - def on_start(self, source: SourceEntry) -> None: - if self._projector is not None: - return - if not self._is_tty: - return - self._current_source = source - self._current_sub_message = "" - self._hide_cursor() - with self._write_lock: - self._spinner_line_active = True - self._lines.claim(stream=self._stream, owner=self) - self._write_spinner_line() - self._start_spinner_loop() - - def on_progress(self, *, source: SourceEntry, message: str) -> None: - if not self._is_tty or self._current_source is None: - return - if source.name != self._current_source.name: - return - self._current_sub_message = message - self._write_spinner_line() - - def on_complete(self, result: LoadExecutionResult) -> None: - canonical_duration_ms: float | None = None - if self._projector is not None: - canonical_duration_ms = self._projector.consume_resource_terminal( - resource_name=result.source_name, - resource_id=f"source:{result.source_name}", - ) - if canonical_duration_ms is None: - return - self._stop_spinner_loop() - if self._is_tty: - with self._write_lock: - self._spinner_line_active = False - self._lines.release(owner=self) - self._stream.write("\r\033[K") - self._stream.flush() - self._show_cursor() - status_text: str = ( - "OK" - if result.status == ExecutionStatus.SUCCESS - else "SKIP" - if result.status == ExecutionStatus.SKIPPED - else "FAIL" - ) - status: str = self._style.status(status=status_text) - duration: str = "" - duration_ms: float | None = ( - result.duration_ms if canonical_duration_ms is None else canonical_duration_ms - ) - if duration_ms is not None: - duration = f"{duration_ms / 1000.0:.2f}s" - rows_loaded: str = f"rows={result.rows_loaded:,}" - ordinal: int = self._source_order[result.source_name] - self._stream.write( - f" {ordinal}/{self._total_count} {result.resource_kind.value:<10}" - f"{result.source_name:<30} {status:<6} {duration} {rows_loaded}\n" - ) - event: LifeCycleEvent - for event in result.lifecycle_events: - if event.kind == LifeCycleEventKind.LOG: - self._write_log_block(event.content) - if result.error_message is not None: - self._stream.write(self._style.error(f" {result.error_message}\n")) - self._stream.flush() - - def clear_transient_line(self) -> None: - """Erase the live spinner row so a persistent line can take its place.""" - - with self._write_lock: - self._stream.write("\r\033[K") - self._stream.flush() - - def redraw_transient_line(self) -> None: - """Draw the live spinner row again below persistent output.""" - - self._write_spinner_line() - - def _write_spinner_line(self) -> None: - with self._write_lock: - if self._spinner_line_active: - self._draw_spinner_line() - - def _draw_spinner_line(self) -> None: - source: SourceEntry | None = self._current_source - if source is None: - return - ordinal: int = self._source_order[source.name] - status: str = self._style.status(status=_ACTIVE_SPINNER_FRAMES[self._spinner_frame_index]) - self._spinner_frame_index = (self._spinner_frame_index + 1) % len(_ACTIVE_SPINNER_FRAMES) - resource_kind: str = ( - "loader" if source.meta.get("sqlbuild_loader_node") is True else "source" - ) - name_display: str = source.name - if self._current_sub_message: - name_display = f"{source.name} {self._current_sub_message}" - line: str = ( - f" {ordinal}/{self._total_count} {resource_kind:<10}{name_display:<48} {status}" - ) - self._stream.write(f"\r\033[K{line}") - self._stream.flush() - - def _start_spinner_loop(self) -> None: - self._stop_spinner_loop() - stop_event: threading.Event = threading.Event() - self._spinner_stop_event = stop_event - spinner_thread: threading.Thread = threading.Thread( - target=self._spin_until_stopped, - args=(stop_event,), - daemon=True, - ) - self._spinner_thread = spinner_thread - spinner_thread.start() - - def _stop_spinner_loop(self) -> None: - if self._spinner_stop_event is not None: - self._spinner_stop_event.set() - if self._spinner_thread is not None and self._spinner_thread.is_alive(): - self._spinner_thread.join(timeout=0.2) - self._spinner_stop_event = None - self._spinner_thread = None - - def _spin_until_stopped(self, stop_event: threading.Event) -> None: - while not stop_event.wait(_SPINNER_TICK_SECONDS): - self._write_spinner_line() - - def _hide_cursor(self) -> None: - if self._cursor_hidden: - return - with self._write_lock: - self._stream.write("\033[?25l") - self._stream.flush() - self._cursor_hidden = True - - def _show_cursor(self) -> None: - if not self._cursor_hidden: - return - with self._write_lock: - self._stream.write("\033[?25h") - self._stream.flush() - self._cursor_hidden = False - - def _write_log_block(self, message: str) -> None: - lines: list[str] = message.splitlines() or [""] - prefix: str = self._style.log_label(" log ") - first_content: str = self._style.muted(lines[0]) - self._stream.write(f"\n{prefix}{first_content}\n") - line: str - for line in lines[1:]: - self._stream.write(f"{self._style.muted(f' {line}')}\n") - - -def format_load_footer( - *, - results: tuple[LoadExecutionResult, ...], - success_count: int, - warn_count: int, - fail_count: int, - skip_count: int, - elapsed: float, - use_color: bool, -) -> str: - style: CliStyle = CliStyle(use_color=use_color) - counts_summary: str = format_summary_footer( - counts=( - ("PASS", success_count), - ("WARN", warn_count), - ("FAIL", fail_count), - ("SKIP", skip_count), - ("TOTAL", len(results)), - ), - use_color=use_color, - elapsed=f"{elapsed:.2f}s", - ) - state: CompletionState = CompletionState.OK - label: str = "Completed successfully" - if fail_count: - state, label = CompletionState.FAIL, "Completed with errors" - elif warn_count: - state, label = CompletionState.WARN, "Completed with warnings" - completion_message: str = format_completion_line( - style=style, state=state, label=label, summary=counts_summary - ) - return "".join(_load_warning_lines(results=results, style=style)) + f"\n{completion_message}\n" - - -def _load_warning_lines(*, results: tuple[LoadExecutionResult, ...], style: CliStyle) -> list[str]: - lines: list[str] = [] - result: LoadExecutionResult - for result in results: - if not result.warning_messages: - continue - if not lines: - lines.append(f"\n{style.warning_strong('Warnings:')}\n\n") - lines.append(f" {result.source_name} (loader {result.loader_name})\n") - lines.extend(f" {message}\n" for message in result.warning_messages) - return lines diff --git a/src/sqlbuild/cli/commands/main/execution/_load.py b/src/sqlbuild/cli/commands/main/execution/_load.py deleted file mode 100644 index a561c754f7..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_load.py +++ /dev/null @@ -1,47 +0,0 @@ -"""CLI load command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.load.execution import execute_load_plan -from sqlbuild.cli.commands._helpers.load.invocation import resolve_load_invocation -from sqlbuild.cli.commands._helpers.load.outputs import ( - resolve_load_exit_code, - write_empty_load_output, - write_load_completion_output, - write_load_execution_header, - write_load_plan_output, - write_load_ready_output, -) -from sqlbuild.cli.commands._helpers.load.planning import prepare_load_execution -from sqlbuild.cli.commands.models import ( - LoadCommandRequest, - LoadExecutionPreparation, - LoadInvocation, - LoadRunOutcome, -) - - -def run_load(request: LoadCommandRequest) -> int: - """Execute the load command.""" - - invocation: LoadInvocation = resolve_load_invocation(request=request) - write_load_ready_output(invocation=invocation) - if not invocation.selected_sources: - write_empty_load_output(request=request, invocation=invocation) - return 0 - preparation: LoadExecutionPreparation = prepare_load_execution( - request=request, - invocation=invocation, - ) - try: - write_load_plan_output(invocation=invocation) - write_load_execution_header(invocation=invocation, preparation=preparation) - outcome: LoadRunOutcome = execute_load_plan( - request=request, - invocation=invocation, - preparation=preparation, - ) - finally: - preparation.provider_session.close() - write_load_completion_output(request=request, invocation=invocation, outcome=outcome) - return resolve_load_exit_code(outcome) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 20537ecb7c..72c900f204 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -36,7 +36,6 @@ CostConfig, ExecutionLimitsConfig, SnapshotsConfig, - SourceEntry, ) if TYPE_CHECKING: @@ -69,7 +68,6 @@ from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult from sqlbuild.executor.diff.models import DiffExecutionResult, FullDiffSizeLimits from sqlbuild.executor.janitor.models import JanitorPlan - from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.integrations.dbt.models import DbtInitRequest from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter @@ -595,64 +593,6 @@ class LoadCommandRequest: json_output_path: Path | None = None -@dataclass(frozen=True) -class LoadInvocation: - """Resolved project, selected sources, and output context for load.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - selected_sources: tuple[SourceEntry, ...] - reference_sources: tuple[SourceEntry, ...] - use_color: bool - progress_stream: TextIO - relation_sources: tuple[SourceEntry, ...] - - -@dataclass(frozen=True) -class LoadExecutionPreparation: - """Prepared adapter, connection, runtime, and execution settings.""" - - adapter_name: str - adapter: BaseAdapter - connection_config: dict[str, object] - target_name: str | None - effective_vars: dict[str, object] - run_id: str - effective_cursor_overrides: CursorOverrides - effective_concurrency: int - provider_session: Any - project_relations: dict[SqlResourceRef, str] | None - - -@dataclass(frozen=True) -class LoadRunOutcome: - """Load execution results, elapsed time, and summary counts.""" - - results: tuple[LoadExecutionResult, ...] - elapsed: float - success_count: int - fail_count: int - skip_count: int - warn_count: int - - -@dataclass(frozen=True) -class LoadSelectionSets: - """Mutable selection sets returned explicitly by selection phases.""" - - selected_sources: set[str] - selected_loaders: set[str] - - -@dataclass(frozen=True) -class LoadSelectorSets: - """Selection sets updated while applying include selectors.""" - - selected_sources: set[str] - selected_loaders: set[str] - directly_selected_loaders: set[str] - - @dataclass(frozen=True) class PlanCommandRequest: """CLI inputs for one plan command invocation.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index a806a80730..b1bbfd5dba 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -13,14 +13,17 @@ from _pytest.monkeypatch import MonkeyPatch from duckdb import DuckDBPyConnection -from sqlbuild._native import discover_project_inputs, format_load_execution_json +from sqlbuild._native import ( + discover_project_inputs, + format_load_execution_json, + run_load_command, + select_load_entries, +) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands._helpers.load.selection import select_load_entries from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.main.execution._audit import run_audit from sqlbuild.cli.commands.main.execution._build import run_build -from sqlbuild.cli.commands.main.execution._load import run_load from sqlbuild.cli.commands.main.execution._scenario import run_scenario from sqlbuild.cli.commands.main.execution._test import run_test from sqlbuild.cli.commands.main.project._plan import run_plan @@ -2049,8 +2052,8 @@ def test_given_source_loader_when_running_load_then_writes_source_table( write_repo_files(tmp_path, test_case.project_files) json_output_path: Path = tmp_path / "target" / "load.json" - exit_code: int = run_load( - LoadCommandRequest( + exit_code: int = run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, select=test_case.select, @@ -3028,7 +3031,9 @@ def test_given_source_loader_write_strategy_when_running_load_twice_then_writes_ write_repo_files(tmp_path, test_case.project_files) for _ in range(test_case.run_count): - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -3403,7 +3408,9 @@ def test_given_loader_cursor_context_has_no_value_when_running_load_then_passes_ connection.close() for _ in range(test_case.run_count): - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -3498,8 +3505,8 @@ def test_given_reload_flag_when_running_load_then_passes_reload_context_to_loade ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load( - LoadCommandRequest( + exit_code: int = run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, reload=test_case.reload, @@ -3547,8 +3554,8 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load( - LoadCommandRequest( + exit_code: int = run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, cursor_overrides=test_case.cursor_overrides, @@ -3891,7 +3898,9 @@ def test_given_generator_loader_with_inferred_columns_when_running_load_then_wri ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -3956,7 +3965,9 @@ def test_given_generator_loader_yields_multiple_rows_when_running_load_then_writ ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -4335,8 +4346,8 @@ def test_given_loader_writes_many_rows_when_running_load_then_formats_human_row_ write_repo_files(tmp_path, test_case.project_files) json_output_path: Path = tmp_path / "target" / "load.json" - exit_code: int = run_load( - LoadCommandRequest( + exit_code: int = run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, json_output_path=json_output_path, @@ -4394,7 +4405,9 @@ def test_given_loader_returns_empty_rows_when_running_load_then_writes_empty_dec ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -4455,7 +4468,9 @@ def test_given_self_managed_loader_when_running_load_then_uses_loader_written_ta write_repo_files(tmp_path, test_case.project_files) for _ in range(test_case.run_count): - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -4558,7 +4573,9 @@ def test_given_loader_returns_conflicting_types_when_running_load_then_fails_cle ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) captured: CaptureResult[str] = capsys.readouterr() assert exit_code == test_case.expected_exit_code @@ -4884,8 +4901,8 @@ def test_given_invalid_load_selectors_when_running_load_then_it_raises_clear_err write_repo_files(tmp_path, test_case.project_files) with pytest.raises(CliUserError) as exc_info: - run_load( - LoadCommandRequest( + run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, select=test_case.select, @@ -4956,8 +4973,8 @@ def test_given_no_selected_managed_sources_when_running_load_then_it_does_not_co ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load( - LoadCommandRequest( + exit_code: int = run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, select=test_case.select, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py index 748dc3ff04..b738f4483d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py @@ -6,8 +6,8 @@ import pytest -from sqlbuild.cli.commands._helpers.load.invocation import _effective_loader_defaults -from sqlbuild.cli.commands._helpers.load.selection import ( +from sqlbuild._native import ( + resolve_loader_defaults, select_load_entries, select_load_reference_entries, ) @@ -150,7 +150,7 @@ def test_given_connection_namespace_when_resolving_load_defaults_then_matches_co local_config=LocalConfig(), ) - defaults: tuple[str | None, str | None] = _effective_loader_defaults( + defaults: tuple[str | None, str | None] = resolve_loader_defaults( discovered_inputs=discovered_inputs, selected_target=None, target_config=None, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index 34d5834104..131bee4ac8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -47,8 +47,8 @@ def build_json_recording_handler( ) -> Callable[..., int]: """Build a fake command handler that records json output flags and returns exit_code.""" - def record_json_handler(*args: object) -> int: - received_args.append(extract_json_output_fields(args)) + def record_json_handler(*args: object, **kwargs: object) -> int: + received_args.append(extract_json_output_fields((*args, *kwargs.values()))) return exit_code return record_json_handler diff --git a/tests/unit/src/sqlbuild/executor/load/test_run.py b/tests/unit/src/sqlbuild/executor/load/test_run.py index 8f75b2c567..ecab9d43a5 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_run.py +++ b/tests/unit/src/sqlbuild/executor/load/test_run.py @@ -8,8 +8,7 @@ import pytest -from sqlbuild._native import NativeProgressProjector -from sqlbuild.cli.commands.classes.load_progress_reporter import LoadProgressReporter +from sqlbuild._native import LoadProgressReporter, NativeProgressProjector from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import SkipMode From 38740b9b17afaf4d7527b9dd2331edd497d9979d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:34:56 +0100 Subject: [PATCH 11/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 9d2b8b7a56..14cd48b092 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 87254 +remaining_lines = 85710 [[stays]] path = "executor" From de555b500c997ded1f525d4b376a88a699ff94f3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:47:43 +0100 Subject: [PATCH 12/37] refactor: register command handler bindings with the CLI bindings --- .../src/bindings/_helpers/cli/registration.rs | 17 +++++++++++++- .../src/bindings/main/register.rs | 22 +++++-------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 5c21cfe7a3..9508da8e0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -1,8 +1,12 @@ -//! Register the native `sqb` argument parsing and command dispatch. +//! Register the native `sqb` argument parsing, command dispatch and command handlers. use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; +use crate::bindings::_helpers::{ + clone, command_runtime, cost_report, dbt_interop, debug_command, freshness, janitor, load, + presentation_text, result_output, workspace_commands, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { argv::register(module)?; @@ -10,5 +14,16 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { progress::register(module)?; progress_reporters::register(module)?; progress_text::register(module)?; + janitor::registration::register(module)?; + freshness::registration::register(module)?; + clone::registration::register(module)?; + load::registration::register(module)?; + result_output::registration::register(module)?; + cost_report::registration::register(module)?; + debug_command::registration::register(module)?; + dbt_interop::registration::register(module)?; + command_runtime::registration::register(module)?; + presentation_text::registration::register(module)?; + workspace_commands::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index aa2a3522b2..d8abc05382 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,12 +5,11 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, - clone, command_runtime, compile_attachment, compile_command, compile_driver, compile_outputs, - compiled_project, contracts, cost_report, cursor, dag_artifact, dbt_interop, debug_command, - discovery, fingerprints, freshness, frontier, graph, janitor, lineage, load, macro_bridge, - manifest, model_config, plan_output, presentation_text, project_assembly, refactor, references, - render, result_output, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, - state_rows, type_system, workspace_commands, + compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, + contracts, cursor, dag_artifact, discovery, fingerprints, frontier, graph, lineage, + macro_bridge, manifest, model_config, plan_output, project_assembly, refactor, references, + render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, + type_system, }; use crate::bindings::models; @@ -45,20 +44,9 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { cache::registration::register(module)?; attachments::registration::register(module)?; refactor::registration::register(module)?; - janitor::registration::register(module)?; - freshness::registration::register(module)?; - clone::registration::register(module)?; - load::registration::register(module)?; plan_output::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; - result_output::registration::register(module)?; - cost_report::registration::register(module)?; - debug_command::registration::register(module)?; - dbt_interop::registration::register(module)?; - command_runtime::registration::register(module)?; - presentation_text::registration::register(module)?; - workspace_commands::registration::register(module)?; auditing::registration::register(module)?; cursor::registration::register(module)?; state_rows::registration::register(module)?; From fc65bd11cf7b6fbb6a057daf72b67ae8ba0cfe54 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:40:53 +0100 Subject: [PATCH 13/37] fix(cli): route progress to stderr for sqb dbt --json --- .../sqlbuild-core/src/progress/constants.rs | 6 ++ .../src/progress/main/machine_output_mode.rs | 21 +++++++ crates/sqlbuild-core/src/progress/main/mod.rs | 1 + crates/sqlbuild-core/src/progress/models.rs | 11 ++++ .../src/progress/tests/test_progress_text.rs | 59 ++++++++++++++++++- .../src/progress/tests/test_types.rs | 6 ++ .../src/bindings/_helpers/cli/progress.rs | 20 ++++++- src/sqlbuild/_native.pyi | 2 + .../commands/_helpers/entry/observability.py | 13 +--- .../cli/commands/main/entry/_test_types.py | 3 + .../main/entry/test_observability_cleanup.py | 31 ++++++++++ 11 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 crates/sqlbuild-core/src/progress/main/machine_output_mode.rs diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index a715a802dc..4338385c18 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -102,3 +102,9 @@ pub(crate) const ERROR_OUTCOME: &str = "error"; /// Inspection SQL longer than this many characters is cut. pub(crate) const SQL_PREVIEW_LENGTH: usize = 160; pub(crate) const EXECUTION_TITLE: &str = "Execution"; +pub(crate) const LINEAGE_COMMAND: &str = "lineage"; +pub(crate) const QUERY_COMMAND: &str = "query"; +pub(crate) const DBT_COMMAND: &str = "dbt"; +pub(crate) const DBT_JSON_OPTION: &str = "--json"; +pub(crate) const JSON_FORMAT: &str = "json"; +pub(crate) const CSV_FORMAT: &str = "csv"; diff --git a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs new file mode 100644 index 0000000000..e483f76b46 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs @@ -0,0 +1,21 @@ +//! Whether a command's stdout carries machine output, so human progress must use stderr. + +use crate::progress::constants::{ + CSV_FORMAT, DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, QUERY_COMMAND, +}; +use crate::progress::models::OutputModeFacts; + +/// `--json`, `sqb dbt ... --json`, `lineage --format json` or `query --format csv|json`. +#[must_use] +pub fn machine_output_mode(facts: &OutputModeFacts) -> bool { + let format = |value: &Option| value.as_deref().unwrap_or_default().to_owned(); + match facts.command.as_deref() { + _ if facts.json => true, + Some(DBT_COMMAND) => facts.dbt_args.iter().any(|arg| arg == DBT_JSON_OPTION), + Some(LINEAGE_COMMAND) => format(&facts.lineage_format) == JSON_FORMAT, + Some(QUERY_COMMAND) => { + [CSV_FORMAT, JSON_FORMAT].contains(&format(&facts.query_format).as_str()) + } + _ => false, + } +} diff --git a/crates/sqlbuild-core/src/progress/main/mod.rs b/crates/sqlbuild-core/src/progress/main/mod.rs index fdbaea3b79..67bb535223 100644 --- a/crates/sqlbuild-core/src/progress/main/mod.rs +++ b/crates/sqlbuild-core/src/progress/main/mod.rs @@ -9,4 +9,5 @@ pub mod format_expectation_detail; pub mod format_expectation_name; pub mod format_inspection_record; pub mod is_planning_completion_message; +pub mod machine_output_mode; pub mod progress_terminal_duration; diff --git a/crates/sqlbuild-core/src/progress/models.rs b/crates/sqlbuild-core/src/progress/models.rs index 7ec2be6f8f..20abcdde23 100644 --- a/crates/sqlbuild-core/src/progress/models.rs +++ b/crates/sqlbuild-core/src/progress/models.rs @@ -142,3 +142,14 @@ pub struct InspectionRecord { pub row_count: Option, pub error: Option, } + +/// The parsed options that decide whether a command writes machine output to stdout. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct OutputModeFacts { + pub json: bool, + pub command: Option, + pub lineage_format: Option, + pub query_format: Option, + /// The arguments after `sqb dbt `, where `--json` selects JSON output. + pub dbt_args: Vec, +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs index 7660f98bcd..c6c45ce34d 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs @@ -6,12 +6,13 @@ use crate::progress::main::format_expectation_detail::format_expectation_detail; use crate::progress::main::format_expectation_name::format_expectation_name; use crate::progress::main::format_inspection_record::format_inspection_record; use crate::progress::main::is_planning_completion_message::is_planning_completion_message; -use crate::progress::models::{ExpectationStep, InspectionRecord}; +use crate::progress::main::machine_output_mode::machine_output_mode; +use crate::progress::models::{ExpectationStep, InspectionRecord, OutputModeFacts}; const PLAIN: CliStyle = CliStyle { use_color: false }; use crate::progress::tests::test_types::{ CompletionMessageTestCase, ExpectationDetailTestCase, InspectionRecordTestCase, - ProgressLineTestCase, + OutputModeTestCase, ProgressLineTestCase, }; #[test] @@ -198,3 +199,57 @@ fn given_inspection_record_when_formatting_then_sql_is_collapsed_and_cut() { ); } } + +#[test] +fn given_parsed_options_when_choosing_output_mode_then_machine_output_is_detected() { + let facts = |command: &str, dbt_args: &[&str]| OutputModeFacts { + command: Some(command.to_owned()), + dbt_args: dbt_args.iter().map(|arg| (*arg).to_owned()).collect(), + ..OutputModeFacts::default() + }; + let test_cases = [ + OutputModeTestCase { + description: "dbt plan with --json", + facts: facts("dbt", &["--select", "orders", "--json"]), + expected_machine_output: true, + }, + OutputModeTestCase { + description: "dbt plan without --json", + facts: facts("dbt", &["--select", "orders"]), + expected_machine_output: false, + }, + OutputModeTestCase { + description: "plan --json", + facts: OutputModeFacts { + json: true, + ..facts("plan", &[]) + }, + expected_machine_output: true, + }, + OutputModeTestCase { + description: "query csv", + facts: OutputModeFacts { + query_format: Some("csv".to_owned()), + ..facts("query", &[]) + }, + expected_machine_output: true, + }, + OutputModeTestCase { + description: "lineage text", + facts: OutputModeFacts { + lineage_format: Some("text".to_owned()), + ..facts("lineage", &[]) + }, + expected_machine_output: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + machine_output_mode(&test_case.facts), + test_case.expected_machine_output, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_types.rs b/crates/sqlbuild-core/src/progress/tests/test_types.rs index d65c94e451..48942a07ce 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_types.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_types.rs @@ -34,3 +34,9 @@ pub(super) struct ProgressLineTestCase { pub(super) line: String, pub(super) expected_line: &'static str, } + +pub(super) struct OutputModeTestCase { + pub(super) description: &'static str, + pub(super) facts: crate::progress::models::OutputModeFacts, + pub(super) expected_machine_output: bool, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs index 50d1c0c5c7..a74d0e08c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs @@ -9,9 +9,10 @@ use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_core::progress::main::claim_progress_terminal::claim_progress_terminal; use sqlbuild_core::progress::main::close_progress_projection::close_progress_projection; use sqlbuild_core::progress::main::consume_progress_event::consume_progress_event; +use sqlbuild_core::progress::main::machine_output_mode::machine_output_mode as output_mode; use sqlbuild_core::progress::main::progress_terminal_duration::progress_terminal_duration; use sqlbuild_core::progress::models::{ - ProgressEvent, ProgressPayload, ProgressProjection, ProgressSettings, + OutputModeFacts, ProgressEvent, ProgressPayload, ProgressProjection, ProgressSettings, }; use sqlbuild_core::terminal_events::models::TerminalRequest; @@ -320,6 +321,22 @@ fn current_native_progress_projector(py: Python<'_>) -> PyResult) -> PyResult { + let mut dbt_args: Vec = Vec::new(); + for arg in args.getattr("dbt_args")?.try_iter()? { + dbt_args.push(arg?.str()?.extract()?); + } + Ok(output_mode(&OutputModeFacts { + json: args.getattr("json")?.is_truthy()?, + command: optional_text(args, "command")?, + lineage_format: optional_text(args, "lineage_format")?, + query_format: optional_text(args, "query_format")?, + dbt_args, + })) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { let py: Python<'_> = module.py(); let keywords: Bound<'_, PyDict> = PyDict::new(py); @@ -331,6 +348,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { .call((CONTEXT_NAME,), Some(&keywords))?, )?; module.add_class::()?; + module.add_function(wrap_pyfunction!(machine_output_mode, module)?)?; module.add_function(wrap_pyfunction!(current_native_progress_projector, module)?) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4c12082f95..78ed46fe0e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -17,6 +17,7 @@ from sqlbuild.adapter.contract.types import ( TypeDialect, ) from sqlbuild.adapter.relations.models import InspectionQueryRecord +from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.models import ( CloneCommandRequest, FreshnessCommandRequest, @@ -1513,6 +1514,7 @@ class NativeProgressProjector: ) -> float | None: ... def close(self) -> None: ... +def machine_output_mode(args: CliNamespace, /) -> bool: ... def current_native_progress_projector() -> NativeProgressProjector | None: ... class ConnectionProgressReporter: diff --git a/src/sqlbuild/cli/commands/_helpers/entry/observability.py b/src/sqlbuild/cli/commands/_helpers/entry/observability.py index 0904aac803..ccfc2f01f9 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/observability.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/observability.py @@ -13,15 +13,13 @@ from sqlbuild._native import ( NativeProgressProjector, TerminalEventIndex, + machine_output_mode, terminal_event_index_scope, ) from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.constants import ( - CSV_OUTPUT_FORMAT, DEBUG_COMMAND, - JSON_OUTPUT_FORMAT, LINEAGE_COMMAND, - QUERY_COMMAND, ) from sqlbuild.cli.commands.main.entrypoint._dispatch_with_output_capture import ( configured_output_capture_scope, @@ -78,14 +76,7 @@ def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> Iterato subscriber=terminal_index.consume, accepts_opaque=False, ) - machine_output: bool = ( - args.json - or (args.command == LINEAGE_COMMAND and args.lineage_format == JSON_OUTPUT_FORMAT) - or ( - args.command == QUERY_COMMAND - and args.query_format in {CSV_OUTPUT_FORMAT, JSON_OUTPUT_FORMAT} - ) - ) + machine_output: bool = machine_output_mode(args) projector: NativeProgressProjector = NativeProgressProjector( stream=sys.stderr if machine_output or args.debug or args.command == DEBUG_COMMAND diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index eb7202473b..f2cd5b643f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -155,3 +155,6 @@ class MachineProgressRoutingCase: json_output: Path | None expected_stdout_fragment: str expected_stderr_fragment: str + command: str | None = None + dbt_args: tuple[str, ...] = () + expected_stdout_excludes: str = "Project compile FAIL" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py index 0d920969dc..30e64d5637 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py @@ -135,6 +135,34 @@ def test_given_no_public_sink_when_observability_runs_then_provider_import_is_sk expected_stdout_fragment="", expected_stderr_fragment="Project compile START", ), + MachineProgressRoutingCase( + description="plan json mode keeps stdout free of progress", + json=True, + json_output=None, + expected_stdout_fragment="", + expected_stderr_fragment="Project compile START", + command="plan", + expected_stdout_excludes="Project compile", + ), + MachineProgressRoutingCase( + description="dbt plan json flag routes progress to stderr", + json=False, + json_output=None, + expected_stdout_fragment="", + expected_stderr_fragment="Project compile START", + command="dbt", + dbt_args=("--json", "--select", "orders"), + expected_stdout_excludes="Project compile", + ), + MachineProgressRoutingCase( + description="dbt plan without json keeps human stdout progress", + json=False, + json_output=None, + expected_stdout_fragment="Project compile START", + expected_stderr_fragment="", + command="dbt", + dbt_args=("--select", "orders"), + ), ), ids=lambda case: case.description, ) @@ -146,6 +174,8 @@ def test_given_json_routing_options_when_operation_runs_then_human_stream_contra args: CliNamespace = CliNamespace() args.json = test_case.json args.json_output = test_case.json_output + args.command = test_case.command + args.dbt_args = list(test_case.dbt_args) with cli_observability_scope(args=args, project_dir=tmp_path): with OperationLifecycle(operation_kind="project", operation_name="project_compile"): @@ -154,6 +184,7 @@ def test_given_json_routing_options_when_operation_runs_then_human_stream_contra captured: CaptureResult[str] = capsys.readouterr() assert test_case.expected_stdout_fragment in captured.out assert test_case.expected_stderr_fragment in captured.err + assert test_case.expected_stdout_excludes not in captured.out @pytest.mark.parametrize( From 895ae09bed533f068c60771450ad3e847d15f2a0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:50:01 +0100 Subject: [PATCH 14/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 14cd48b092..bff58fa5e8 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 85710 +remaining_lines = 85701 [[stays]] path = "executor" From d2d788a493ab5fe6312682d715fcabc672749c6f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:15:02 +0100 Subject: [PATCH 15/37] refactor(diagnostics): redact diagnostic secrets natively Secret redaction of diagnostic text and structured logging fields runs in sqlbuild-core with Rust regex. The Python logging handlers, formatters, filters and the LogRecord field collection stay in Python and are listed as stays in python-boundary.toml. --- Cargo.lock | 3 + crates/sqlbuild-core/Cargo.toml | 3 + crates/sqlbuild-core/src/lib.rs | 1 + .../src/redaction/_helpers/mod.rs | 1 + .../src/redaction/_helpers/patterns.rs | 37 ++++++++ .../sqlbuild-core/src/redaction/constants.rs | 33 ++++++++ .../sqlbuild-core/src/redaction/main/mod.rs | 2 + .../src/redaction/main/redact_text.rs | 15 ++++ .../src/redaction/main/secret_field_name.rs | 10 +++ crates/sqlbuild-core/src/redaction/mod.rs | 6 ++ crates/sqlbuild-core/src/redaction/tests.rs | 4 + .../src/redaction/tests/test_redact_text.rs | 32 +++++++ .../src/redaction/tests/test_types.rs | 5 ++ .../src/bindings/_helpers/diagnostics/mod.rs | 1 + .../_helpers/diagnostics/redaction.rs | 84 +++++++++++++++++++ .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/main/register.rs | 12 +-- python-boundary.toml | 40 +++++++++ src/sqlbuild/_native.pyi | 2 + .../diagnostics/_helpers/redaction.py | 64 +------------- .../classes/diagnostic_record_redactor.py | 7 +- 21 files changed, 294 insertions(+), 69 deletions(-) create mode 100644 crates/sqlbuild-core/src/redaction/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/redaction/_helpers/patterns.rs create mode 100644 crates/sqlbuild-core/src/redaction/constants.rs create mode 100644 crates/sqlbuild-core/src/redaction/main/mod.rs create mode 100644 crates/sqlbuild-core/src/redaction/main/redact_text.rs create mode 100644 crates/sqlbuild-core/src/redaction/main/secret_field_name.rs create mode 100644 crates/sqlbuild-core/src/redaction/mod.rs create mode 100644 crates/sqlbuild-core/src/redaction/tests.rs create mode 100644 crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs create mode 100644 crates/sqlbuild-core/src/redaction/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs diff --git a/Cargo.lock b/Cargo.lock index 7630404f05..44860e0dd9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1210,6 +1210,9 @@ dependencies = [ [[package]] name = "sqlbuild-core" version = "0.156.1" +dependencies = [ + "regex", +] [[package]] name = "sqlbuild-cursor" diff --git a/crates/sqlbuild-core/Cargo.toml b/crates/sqlbuild-core/Cargo.toml index c6657c84e0..2d235db809 100644 --- a/crates/sqlbuild-core/Cargo.toml +++ b/crates/sqlbuild-core/Cargo.toml @@ -7,5 +7,8 @@ license.workspace = true publish.workspace = true rust-version.workspace = true +[dependencies] +regex.workspace = true + [lints] workspace = true diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 196d5a730a..f63645fa4f 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -16,6 +16,7 @@ pub mod models; pub mod panics; pub mod posix_path; pub mod progress; +pub mod redaction; pub mod terminal_events; pub mod text; diff --git a/crates/sqlbuild-core/src/redaction/_helpers/mod.rs b/crates/sqlbuild-core/src/redaction/_helpers/mod.rs new file mode 100644 index 0000000000..d0c4d5d890 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod patterns; diff --git a/crates/sqlbuild-core/src/redaction/_helpers/patterns.rs b/crates/sqlbuild-core/src/redaction/_helpers/patterns.rs new file mode 100644 index 0000000000..69f654ab16 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/_helpers/patterns.rs @@ -0,0 +1,37 @@ +//! The four secret patterns, compiled once per process. + +use std::sync::LazyLock; + +use regex::{Regex, RegexBuilder}; + +use crate::redaction::constants::{ + AUTHORIZATION_PATTERN, BEARER_PATTERN, PATTERN_SIZE_LIMIT, SECRET_VALUE_PATTERN, + URI_CREDENTIAL_PATTERN, +}; + +pub(crate) static AUTHORIZATION: LazyLock> = + LazyLock::new(|| compiled(AUTHORIZATION_PATTERN)); +pub(crate) static BEARER: LazyLock> = + LazyLock::new(|| compiled(BEARER_PATTERN)); +pub(crate) static SECRET_VALUE: LazyLock> = + LazyLock::new(|| compiled(SECRET_VALUE_PATTERN)); +pub(crate) static URI_CREDENTIAL: LazyLock> = + LazyLock::new(|| compiled(URI_CREDENTIAL_PATTERN)); + +fn compiled(pattern: &str) -> Result { + RegexBuilder::new(pattern) + .size_limit(PATTERN_SIZE_LIMIT) + .build() +} + +/// `pattern.sub(replacement, text)`; a pattern that failed to compile matches nothing. +pub(crate) fn substituted( + pattern: &LazyLock>, + text: &str, + replacement: &str, +) -> String { + match pattern.as_ref() { + Ok(regex) => regex.replace_all(text, replacement).into_owned(), + Err(_) => text.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/redaction/constants.rs b/crates/sqlbuild-core/src/redaction/constants.rs new file mode 100644 index 0000000000..6b823c8bb1 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/constants.rs @@ -0,0 +1,33 @@ +//! Secret field names and the replacement diagnostic redaction writes. + +pub(crate) const SECRET_FIELD_PARTS: [&str; 11] = [ + "access_key", + "api_key", + "authorization", + "client_secret", + "credential", + "dsn", + "password", + "private_key", + "refresh_token", + "secret", + "token", +]; +pub(crate) const SECRET_VALUE_PATTERN: &str = concat!( + r"(?i)\b(api[-_]?key|access[-_]?key|private[-_]?key|authorization|", + r"client[-_]?secret|refresh[-_]?token|", + r"password|passwd|pwd|secret|token|dsn|connection_string)", + r#"(\s*[=:]\s*)("[^"\r\n]{0,512}"|'[^'\r\n]{0,512}'|[^\s,;]{1,512})"#, +); +pub(crate) const AUTHORIZATION_PATTERN: &str = concat!( + r"(?i)\b(authorization)(\s*[=:]\s*)(?:basic|bearer)\s+", + r#"("[^"\r\n]{1,512}"|'[^'\r\n]{1,512}'|[^\s,;]{1,512})"#, +); +pub(crate) const BEARER_PATTERN: &str = r"(?i)\bbearer\s+[a-z0-9._~+/=-]{1,512}"; +pub(crate) const URI_CREDENTIAL_PATTERN: &str = + r"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@]+:)([^\s@]+)(@)"; +pub(crate) const PAIR_REPLACEMENT: &str = "${1}${2}[REDACTED]"; +pub(crate) const BEARER_REPLACEMENT: &str = "Bearer [REDACTED]"; +pub(crate) const URI_REPLACEMENT: &str = "${1}[REDACTED]${3}"; +/// Compiled size allowance for the bounded Unicode repetitions above. +pub(crate) const PATTERN_SIZE_LIMIT: usize = 1 << 28; diff --git a/crates/sqlbuild-core/src/redaction/main/mod.rs b/crates/sqlbuild-core/src/redaction/main/mod.rs new file mode 100644 index 0000000000..47aff8ca43 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/main/mod.rs @@ -0,0 +1,2 @@ +pub mod redact_text; +pub mod secret_field_name; diff --git a/crates/sqlbuild-core/src/redaction/main/redact_text.rs b/crates/sqlbuild-core/src/redaction/main/redact_text.rs new file mode 100644 index 0000000000..9764c88a0e --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/main/redact_text.rs @@ -0,0 +1,15 @@ +//! `redact_text`: obvious named secrets and URI passwords removed from diagnostic text. + +use crate::redaction::_helpers::patterns::{ + AUTHORIZATION, BEARER, SECRET_VALUE, URI_CREDENTIAL, substituted, +}; +use crate::redaction::constants::{BEARER_REPLACEMENT, PAIR_REPLACEMENT, URI_REPLACEMENT}; + +/// Authorization headers, bearer tokens, `name=value` secrets, then URI passwords redacted. +#[must_use] +pub fn redact_text(value: &str) -> String { + let authorization: String = substituted(&AUTHORIZATION, value, PAIR_REPLACEMENT); + let bearer: String = substituted(&BEARER, &authorization, BEARER_REPLACEMENT); + let named: String = substituted(&SECRET_VALUE, &bearer, PAIR_REPLACEMENT); + substituted(&URI_CREDENTIAL, &named, URI_REPLACEMENT) +} diff --git a/crates/sqlbuild-core/src/redaction/main/secret_field_name.rs b/crates/sqlbuild-core/src/redaction/main/secret_field_name.rs new file mode 100644 index 0000000000..3895df24be --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/main/secret_field_name.rs @@ -0,0 +1,10 @@ +//! `secret_field_name`: whether a structured logging field holds a secret by its name. + +use crate::redaction::constants::SECRET_FIELD_PARTS; + +/// Whether the lowercased name, with `-` read as `_`, contains a secret field part. +#[must_use] +pub fn secret_field_name(name: &str) -> bool { + let lowered: String = name.to_lowercase().replace('-', "_"); + SECRET_FIELD_PARTS.iter().any(|part| lowered.contains(part)) +} diff --git a/crates/sqlbuild-core/src/redaction/mod.rs b/crates/sqlbuild-core/src/redaction/mod.rs new file mode 100644 index 0000000000..37f9081f4f --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/redaction/tests.rs b/crates/sqlbuild-core/src/redaction/tests.rs new file mode 100644 index 0000000000..bb9efc8856 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_redact_text.rs"] +mod redact_text; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs b/crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs new file mode 100644 index 0000000000..a998e3bc7c --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs @@ -0,0 +1,32 @@ +use crate::redaction::main::redact_text::redact_text; +use crate::redaction::tests::test_types::RedactTestCase; + +#[test] +fn given_diagnostic_text_when_redacting_then_matches_python_substitutions() { + let test_cases = [ + RedactTestCase { + description: "authorization header keeps its scheme separator", + text: "Authorization: Bearer abc.def", + expected_text: "Authorization: [REDACTED]", + }, + RedactTestCase { + description: "named secrets and URI passwords", + text: "password = 'hunter2', url=postgres://user:pw@host/db", + expected_text: "password = [REDACTED], url=postgres://user:[REDACTED]@host/db", + }, + RedactTestCase { + description: "plain text is unchanged", + text: "nothing to see", + expected_text: "nothing to see", + }, + ]; + + for test_case in test_cases { + assert_eq!( + redact_text(test_case.text), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/redaction/tests/test_types.rs b/crates/sqlbuild-core/src/redaction/tests/test_types.rs new file mode 100644 index 0000000000..7466e971e7 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(crate) struct RedactTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) expected_text: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs new file mode 100644 index 0000000000..5dfdec4811 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs @@ -0,0 +1 @@ +pub(crate) mod redaction; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs new file mode 100644 index 0000000000..603adb3049 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs @@ -0,0 +1,84 @@ +//! Secret-safe diagnostic text and structured logging field values. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyStringMethods, PyTuple, + PyTypeMethods, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::redaction::main::redact_text::redact_text; +use sqlbuild_core::redaction::main::secret_field_name::secret_field_name; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const REDACTED: &str = "[REDACTED]"; + +/// Obvious named secrets and URI passwords redacted from diagnostic text. +#[pyfunction] +#[pyo3(signature = (value))] +fn redact_diagnostic_text(value: &str) -> String { + redact_text(value) +} + +fn redacted_mapping<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let redacted: Bound<'py, PyDict> = PyDict::new(py); + for item in value.call_method0("items")?.try_iter()? { + let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let name: String = key.str()?.extract()?; + redacted.set_item(&name, redacted_value(&name, &item)?)?; + } + Ok(redacted.into_any()) +} + +/// A persistence-safe value for one structured logging field. +fn redacted_value<'py>(name: &str, value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if secret_field_name(name) { + return Ok(PyString::new(py, REDACTED).into_any()); + } + if let Ok(text) = value.downcast::() { + return Ok(PyString::new(py, &redact_text(text.to_str()?)).into_any()); + } + let scalar: bool = value.is_none() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::(); + if scalar { + return Ok(value.clone()); + } + if value.is_instance(&imported(py, "collections.abc", "Mapping")?)? { + return redacted_mapping(value); + } + if value.is_instance_of::() || value.is_instance_of::() { + let items: Vec> = value + .try_iter()? + .map(|item| redacted_value(name, &item?)) + .collect::>()?; + return Ok(PyList::new(py, items)?.into_any()); + } + let rendered: PyResult = value.str().and_then(|text| text.extract()); + Ok(match rendered { + Ok(text) => PyString::new(py, &redact_text(&text)).into_any(), + Err(_) => { + let type_name: String = value.get_type().name()?.extract()?; + PyString::new(py, &format!("<{type_name}>")).into_any() + } + }) +} + +/// A persistence-safe value for one structured logging field. +#[pyfunction] +#[pyo3(signature = (*, name, value))] +fn redact_diagnostic_value<'py>( + name: &str, + value: &Bound<'py, PyAny>, +) -> PyResult> { + redacted_value(name, value) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(redact_diagnostic_text, module)?)?; + module.add_function(wrap_pyfunction!(redact_diagnostic_value, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 6af3bb6d14..abcc940d0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -19,6 +19,7 @@ pub(crate) mod cursor; pub(crate) mod dag_artifact; pub(crate) mod dbt_interop; pub(crate) mod debug_command; +pub(crate) mod diagnostics; pub(crate) mod discovery; pub(crate) mod fingerprints; pub(crate) mod freshness; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index d8abc05382..4e7e172b80 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,11 +5,12 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, - compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, - contracts, cursor, dag_artifact, discovery, fingerprints, frontier, graph, lineage, - macro_bridge, manifest, model_config, plan_output, project_assembly, refactor, references, - render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, - type_system, + clone, command_runtime, compile_attachment, compile_command, compile_driver, compile_outputs, + compiled_project, contracts, cost_report, cursor, dag_artifact, debug_command, diagnostics, + discovery, fingerprints, freshness, frontier, graph, janitor, lineage, macro_bridge, manifest, + model_config, plan_output, presentation_text, project_assembly, refactor, references, render, + result_output, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, + state_rows, type_system, workspace_commands, }; use crate::bindings::models; @@ -50,6 +51,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { auditing::registration::register(module)?; cursor::registration::register(module)?; state_rows::registration::register(module)?; + diagnostics::redaction::register(module)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index bff58fa5e8..6cbd9ef0f8 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -59,6 +59,46 @@ path = "rules" reason = "public-api" note = "Public custom-rule API and test helper." +[[stays]] +path = "diagnostics/_helpers/logging.py" +reason = "public-api" +note = "Installs SQLBuild's handlers on Python's logging, which users and integrations configure." + +[[stays]] +path = "diagnostics/_helpers/logging_formatters.py" +reason = "public-api" +note = "Python logging formatters for console and file diagnostics." + +[[stays]] +path = "diagnostics/_helpers/redaction.py" +reason = "public-api" +note = "Collects redacted extra fields from a Python LogRecord; the redaction itself is native." + +[[stays]] +path = "diagnostics/classes/diagnostic_record_redactor.py" +reason = "public-api" +note = "Redaction entry points that Python logging handlers call per record." + +[[stays]] +path = "diagnostics/classes/diagnostics_console_formatter.py" +reason = "public-api" +note = "A logging.Formatter subclass for the console stream." + +[[stays]] +path = "diagnostics/classes/dynamic_stderr_handler.py" +reason = "public-api" +note = "A logging.Handler that follows the current sys.stderr." + +[[stays]] +path = "diagnostics/classes/invocation_diagnostic_routing.py" +reason = "public-api" +note = "Attaches and removes per-invocation logging handlers." + +[[stays]] +path = "diagnostics/classes/log_record_route_filter.py" +reason = "public-api" +note = "A logging.Filter that routes records to destinations." + [[stays]] path = "rule_engine/_helpers/host" reason = "user-code" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 78ed46fe0e..744598a126 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3079,6 +3079,8 @@ def evaluate_source_freshness_age_policy( data_version: object, observed_at: datetime, ) -> SourceFreshnessAgeStatus | None: ... +def redact_diagnostic_text(value: str) -> str: ... +def redact_diagnostic_value(*, name: str, value: object) -> object: ... def cursor_parse(*, raw: object, cursor_type: str) -> CursorScalar: ... def cursor_try_parse(*, raw: object, cursor_type: str) -> CursorScalar | None: ... def cursor_render(*, value: CursorScalar) -> str: ... diff --git a/src/sqlbuild/diagnostics/_helpers/redaction.py b/src/sqlbuild/diagnostics/_helpers/redaction.py index 2a522b3164..169d967780 100644 --- a/src/sqlbuild/diagnostics/_helpers/redaction.py +++ b/src/sqlbuild/diagnostics/_helpers/redaction.py @@ -1,79 +1,21 @@ -"""Secret-safe diagnostic record rendering.""" +"""Redacted structured fields of one logging record.""" from __future__ import annotations import logging -import re -from collections.abc import Mapping from typing import Final +from sqlbuild._native import redact_diagnostic_value from sqlbuild.diagnostics.constants import FORMATTER_RECORD_FIELDS, SQL_TEXT_FIELD -_REDACTED: Final[str] = "[REDACTED]" -_SECRET_FIELD_PARTS: Final[tuple[str, ...]] = ( - "access_key", - "api_key", - "authorization", - "client_secret", - "credential", - "dsn", - "password", - "private_key", - "refresh_token", - "secret", - "token", -) -_SECRET_VALUE_PATTERN: Final[re.Pattern[str]] = re.compile( - r"(?i)\b(api[-_]?key|access[-_]?key|private[-_]?key|authorization|" - r"client[-_]?secret|refresh[-_]?token|" - r"password|passwd|pwd|secret|token|dsn|connection_string)" - r"(\s*[=:]\s*)(\"[^\"\r\n]{0,512}\"|'[^'\r\n]{0,512}'|[^\s,;]{1,512})" -) -_AUTHORIZATION_PATTERN: Final[re.Pattern[str]] = re.compile( - r"(?i)\b(authorization)(\s*[=:]\s*)(?:basic|bearer)\s+" - r"(\"[^\"\r\n]{1,512}\"|'[^'\r\n]{1,512}'|[^\s,;]{1,512})" -) -_BEARER_PATTERN: Final[re.Pattern[str]] = re.compile(r"(?i)\bbearer\s+[a-z0-9._~+/=-]{1,512}") -_URI_CREDENTIAL_PATTERN: Final[re.Pattern[str]] = re.compile( - r"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@]+:)([^\s@]+)(@)" -) _STANDARD_RECORD_FIELDS: Final[frozenset[str]] = frozenset(logging.makeLogRecord({}).__dict__) -def redact_text(value: str) -> str: - """Redact obvious named secrets and URI passwords from diagnostic text.""" - - authorization_redacted: str = _AUTHORIZATION_PATTERN.sub(rf"\1\2{_REDACTED}", value) - bearer_redacted: str = _BEARER_PATTERN.sub(f"Bearer {_REDACTED}", authorization_redacted) - named_redacted: str = _SECRET_VALUE_PATTERN.sub(rf"\1\2{_REDACTED}", bearer_redacted) - return _URI_CREDENTIAL_PATTERN.sub(rf"\1{_REDACTED}\3", named_redacted) - - -def redact_value(*, name: str, value: object) -> object: - """Return a persistence-safe value for one structured logging field.""" - - lowered_name: str = name.lower().replace("-", "_") - if any(part in lowered_name for part in _SECRET_FIELD_PARTS): - return _REDACTED - if isinstance(value, str): - return redact_text(value) - if value is None or isinstance(value, (bool, int, float)): - return value - if isinstance(value, Mapping): - return {str(key): redact_value(name=str(key), value=item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [redact_value(name=name, value=item) for item in value] - try: - return redact_text(str(value)) - except Exception: - return f"<{type(value).__name__}>" - - def safe_extra_fields(record: logging.LogRecord) -> dict[str, object]: """Collect redacted non-standard fields without evaluating arbitrary formatters.""" return { - key: redact_value(name=key, value=value) + key: redact_diagnostic_value(name=key, value=value) for key, value in record.__dict__.items() if key not in _STANDARD_RECORD_FIELDS and key not in FORMATTER_RECORD_FIELDS diff --git a/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py b/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py index 026d7ea792..d7a4a12f56 100644 --- a/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py +++ b/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py @@ -4,7 +4,8 @@ import logging -from sqlbuild.diagnostics._helpers.redaction import redact_text, redact_value, safe_extra_fields +from sqlbuild._native import redact_diagnostic_text, redact_diagnostic_value +from sqlbuild.diagnostics._helpers.redaction import safe_extra_fields class DiagnosticRecordRedactor: @@ -14,13 +15,13 @@ class DiagnosticRecordRedactor: def text(value: str) -> str: """Redact obvious secrets from diagnostic text.""" - return redact_text(value) + return redact_diagnostic_text(value) @staticmethod def value(*, name: str, value: object) -> object: """Redact one structured field value.""" - return redact_value(name=name, value=value) + return redact_diagnostic_value(name=name, value=value) @staticmethod def extras(record: logging.LogRecord) -> dict[str, object]: From 85403dfde9250c8da809078263f0cf143862134c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:37:15 +0100 Subject: [PATCH 16/37] refactor(rules): key and look up the SQL rule result cache natively SQL rule cache keys (lint settings, rule codes, model SQL and dynamic column contract, non-model file text), stored-entry lookup and the cache file now live in sqlbuild-rules as pure functions with a new key format (sql-rules-v6), so the first run after upgrading misses once. A new integration test pins cold misses, warm hits and invalidation on model, threshold, exception-policy and rule-selection edits. --- .../src/bindings/_helpers/rules/mod.rs | 1 + .../bindings/_helpers/rules/registration.rs | 3 +- .../src/bindings/_helpers/rules/rule_cache.rs | 191 ++++++++++++++ crates/sqlbuild-rules/src/lib.rs | 1 + .../src/rule_cache/_helpers/digest.rs | 19 ++ .../src/rule_cache/_helpers/mod.rs | 1 + .../src/rule_cache/constants.rs | 15 ++ .../src/rule_cache/main/file_rule_key.rs | 23 ++ .../src/rule_cache/main/lint_key.rs | 35 +++ .../src/rule_cache/main/lookup_rule_cache.rs | 19 ++ .../sqlbuild-rules/src/rule_cache/main/mod.rs | 6 + .../src/rule_cache/main/model_rule_key.rs | 24 ++ .../src/rule_cache/main/parse_rule_cache.rs | 35 +++ .../src/rule_cache/main/rule_cache_text.rs | 27 ++ crates/sqlbuild-rules/src/rule_cache/mod.rs | 5 + .../sqlbuild-rules/src/rule_cache/models.rs | 35 +++ crates/sqlbuild-rules/src/rule_cache/types.rs | 8 + src/sqlbuild/_native.pyi | 13 + .../rule_engine/_helpers/run/rules.py | 233 +++++------------- .../sqlbuild/cli/commands/main/_test_types.py | 9 + .../sqlbuild/cli/commands/main/test_rules.py | 83 ++++++- 21 files changed, 606 insertions(+), 180 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/constants.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/main/file_rule_key.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/main/lookup_rule_cache.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/main/mod.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/main/parse_rule_cache.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/main/rule_cache_text.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/mod.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/models.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/types.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 9956d91032..9c49a5d267 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -6,4 +6,5 @@ pub(crate) mod registration; pub(crate) mod relation_keys; pub(crate) mod results; pub(crate) mod rows; +pub(crate) mod rule_cache; pub(crate) mod skills; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index a9de0613e5..a1fbcc367e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ - custom_rule_evidence, dsl_headers, evaluation, lint, results, skills, + custom_rule_evidence, dsl_headers, evaluation, lint, results, rule_cache, skills, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -11,6 +11,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { lint::register(module)?; dsl_headers::register(module)?; custom_rule_evidence::register(module)?; + rule_cache::register(module)?; results::register(module)?; skills::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs new file mode 100644 index 0000000000..cccd043426 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -0,0 +1,191 @@ +//! SQL rule result cache keys, stored-entry lookup and the cache file. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use serde_json::{Value, json}; +use sqlbuild_rules::rule_cache::main::file_rule_key::file_rule_key; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use sqlbuild_rules::rule_cache::main::lint_key::lint_key; +use sqlbuild_rules::rule_cache::main::lookup_rule_cache::lookup_rule_cache; +use sqlbuild_rules::rule_cache::main::model_rule_key::model_rule_key; +use sqlbuild_rules::rule_cache::main::parse_rule_cache::parse_rule_cache; +use sqlbuild_rules::rule_cache::main::rule_cache_text::rule_cache_text; +use sqlbuild_rules::rule_cache::models::{CacheLookup, CachedEntry, LintKeyInput, ModelKeyInput}; +use sqlbuild_rules::rule_cache::types::CachedEntries; + +/// Stored entries as `{path: (key, findings JSON)}`. +type StoredEntries = BTreeMap; + +fn invalid(error: serde_json::Error) -> PyErr { + PyValueError::new_err(error.to_string()) +} + +/// The lint settings part of every SQL rule cache key. +#[pyfunction] +#[pyo3(signature = (*, config))] +fn sql_rule_lint_key(config: &Bound<'_, PyAny>) -> PyResult { + let mut relation_keys: Vec<(String, Vec>)> = Vec::new(); + for item in config + .getattr("relation_keys")? + .call_method0("items")? + .try_iter()? + { + relation_keys.push(item?.extract()?); + } + Ok(lint_key(&LintKeyInput { + dialect: config.getattr("dialect")?.extract()?, + max_literal_length: config.getattr("max_literal_length")?.extract()?, + max_ranking_order_by: config.getattr("max_ranking_order_by")?.extract()?, + relation_keys, + allow_suppressions: config.getattr("allow_suppressions")?.extract()?, + })) +} + +/// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. +#[pyfunction] +#[pyo3(signature = (*, lint_key, codes, models))] +fn sql_model_rule_keys( + lint_key: &str, + codes: Vec, + models: Vec<(String, String, String, Option)>, +) -> BTreeMap { + models + .iter() + .map(|(path, authored_sql, query_sql, contract)| { + let model = ModelKeyInput { + authored_sql, + query_sql, + contract: contract.as_deref(), + }; + (path.clone(), model_rule_key(lint_key, &codes, &model)) + }) + .collect() +} + +/// One non-model SQL file's cache key, or `None` when its findings depend on expansion. +#[pyfunction] +#[pyo3(signature = (**request))] +fn sql_file_rule_key(request: Option<&Bound<'_, PyDict>>) -> PyResult> { + let request = Request::new("sql_file_rule_key", request)?; + let lint_key: String = request.get("lint_key")?.extract()?; + let codes: Vec = request.get("codes")?.extract()?; + let relative_path: String = request.get("relative_path")?.extract()?; + let contents: String = request.get("contents")?.extract()?; + let expanded: bool = request.get("expanded")?.extract()?; + Ok(file_rule_key( + &lint_key, + &codes, + (&relative_path, &contents), + expanded, + )) +} + +/// The usable stored entries of the cache file at `path`; unreadable files read as empty. +#[pyfunction] +#[pyo3(signature = (*, path))] +fn load_sql_rule_cache(path: PathBuf) -> PyResult { + let Ok(text) = std::fs::read_to_string(&path) else { + return Ok(StoredEntries::new()); + }; + parse_rule_cache(&text) + .into_iter() + .map(|(key, entry)| { + let findings: String = serde_json::to_string(&entry.findings).map_err(invalid)?; + Ok((key, (entry.identity, findings))) + }) + .collect() +} + +fn cached_entries(entries: &StoredEntries) -> PyResult { + entries + .iter() + .map(|(path, (identity, findings))| { + let findings: Vec = serde_json::from_str(findings).map_err(invalid)?; + let entry = CachedEntry { + identity: identity.clone(), + findings, + }; + Ok((path.clone(), entry)) + }) + .collect() +} + +/// `(hits, misses)`: stored findings JSON of paths whose key matches, and every other path. +#[pyfunction] +#[pyo3(signature = (*, entries, keys))] +fn lookup_sql_rule_cache( + entries: StoredEntries, + keys: Vec<(String, String)>, +) -> PyResult<(BTreeMap, Vec)> { + let lookup: CacheLookup = lookup_rule_cache(&cached_entries(&entries)?, &keys); + let hits: BTreeMap = lookup + .hits + .into_iter() + .map(|(path, findings)| Ok((path, serde_json::to_string(&findings).map_err(invalid)?))) + .collect::>()?; + Ok((hits, lookup.misses)) +} + +fn finding_payload(finding: &Bound<'_, PyAny>) -> PyResult { + let path: String = finding + .getattr("path")? + .call_method0("as_posix")? + .extract()?; + Ok(json!({ + "unevaluated": finding.getattr("unevaluated")?.extract::()?, + "code": finding.getattr("code")?.extract::()?, + "path": path, + "line": finding.getattr("line")?.extract::>()?, + "column": finding.getattr("column")?.extract::>()?, + "message": finding.getattr("message")?.extract::()?, + "remediation": finding.getattr("remediation")?.extract::>()?, + "fixable": finding.getattr("fixable")?.extract::()?, + })) +} + +/// The stored JSON of one path's findings. +#[pyfunction] +#[pyo3(signature = (*, findings))] +fn encode_sql_rule_findings(findings: Vec>) -> PyResult { + let payloads: Vec = findings + .iter() + .map(finding_payload) + .collect::>()?; + serde_json::to_string(&payloads).map_err(invalid) +} + +/// Replace the cache file at `path` atomically with `entries`. +#[pyfunction] +#[pyo3(signature = (*, path, entries))] +fn write_sql_rule_cache(path: PathBuf, entries: &Bound<'_, PyDict>) -> PyResult<()> { + let mut stored = StoredEntries::new(); + for (key, value) in entries.iter() { + stored.insert(key.extract()?, value.extract()?); + } + let text: String = rule_cache_text(&cached_entries(&stored)?); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let temporary: PathBuf = path.with_extension(format!("tmp-{}", std::process::id())); + std::fs::write(&temporary, text)?; + std::fs::rename(&temporary, &path)?; + Ok(()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_rule_lint_key, module)?)?; + module.add_function(wrap_pyfunction!(sql_model_rule_keys, module)?)?; + module.add_function(wrap_pyfunction!(sql_file_rule_key, module)?)?; + module.add_function(wrap_pyfunction!(load_sql_rule_cache, module)?)?; + module.add_function(wrap_pyfunction!(lookup_sql_rule_cache, module)?)?; + module.add_function(wrap_pyfunction!(encode_sql_rule_findings, module)?)?; + module.add_function(wrap_pyfunction!(write_sql_rule_cache, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index 49dca7ea3c..d6754baad4 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -8,6 +8,7 @@ pub mod dsl_headers; pub mod engine; pub mod errors; pub mod models; +pub mod rule_cache; pub mod rule_results; pub mod rules; pub mod scope_metadata; diff --git a/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs b/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs new file mode 100644 index 0000000000..50033d3c68 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs @@ -0,0 +1,19 @@ +//! SHA-256 over length-prefixed fields, so no two field lists share a key. + +use sha2::{Digest, Sha256}; + +use crate::constants::NATIVE_BUILD_IDENTITY; +use crate::rule_cache::constants::CACHE_VERSION; + +/// The hex digest of the cache version, the native build and each field in order. +pub(crate) fn field_digest(kind: &str, fields: &[&str]) -> String { + let mut digest = Sha256::new(); + for field in [CACHE_VERSION, NATIVE_BUILD_IDENTITY, kind] + .iter() + .chain(fields.iter()) + { + digest.update((field.len() as u64).to_le_bytes()); + digest.update(field.as_bytes()); + } + format!("{:x}", digest.finalize()) +} diff --git a/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs b/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs new file mode 100644 index 0000000000..71d61e1a07 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod digest; diff --git a/crates/sqlbuild-rules/src/rule_cache/constants.rs b/crates/sqlbuild-rules/src/rule_cache/constants.rs new file mode 100644 index 0000000000..4e3c6cd34a --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/constants.rs @@ -0,0 +1,15 @@ +//! The SQL rule result cache format and the inputs its keys separate. + +/// Bumped whenever key derivation or the stored finding shape changes. +pub(crate) const CACHE_VERSION: &str = "sql-rules-v6"; +pub(crate) const MODEL_KEY_KIND: &str = "model"; +pub(crate) const FILE_KEY_KIND: &str = "file"; +pub(crate) const LINT_KEY_KIND: &str = "lint"; +pub(crate) const NO_CONTRACT: &str = ""; +pub(crate) const MACRO_TOKEN: &str = "@"; +pub(crate) const TEMPLATE_INTERPOLATION_START: &str = "${"; +pub(crate) const VERSION_KEY: &str = "version"; +pub(crate) const BUILD_IDENTITY_KEY: &str = "build_identity"; +pub(crate) const ENTRIES_KEY: &str = "entries"; +pub(crate) const IDENTITY_KEY: &str = "identity"; +pub(crate) const FINDINGS_KEY: &str = "findings"; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/file_rule_key.rs b/crates/sqlbuild-rules/src/rule_cache/main/file_rule_key.rs new file mode 100644 index 0000000000..ffe137298a --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/file_rule_key.rs @@ -0,0 +1,23 @@ +//! `file_rule_key`: the cache key of one non-model SQL file's rule findings. + +use crate::rule_cache::_helpers::digest::field_digest; +use crate::rule_cache::constants::{FILE_KEY_KIND, MACRO_TOKEN, TEMPLATE_INTERPOLATION_START}; + +/// A digest of the lint key, codes, path and text; `None` when macros or templates expand it. +#[must_use] +pub fn file_rule_key( + lint_key: &str, + codes: &[String], + (relative_path, contents): (&str, &str), + expanded: bool, +) -> Option { + if expanded || contents.contains(MACRO_TOKEN) || contents.contains(TEMPLATE_INTERPOLATION_START) + { + return None; + } + let codes: String = codes.join("\0"); + Some(field_digest( + FILE_KEY_KIND, + &[lint_key, &codes, relative_path, contents], + )) +} diff --git a/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs b/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs new file mode 100644 index 0000000000..8f5b8bdc4d --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs @@ -0,0 +1,35 @@ +//! `lint_key`: the lint settings part of every SQL rule cache key. + +use crate::rule_cache::_helpers::digest::field_digest; +use crate::rule_cache::constants::LINT_KEY_KIND; +use crate::rule_cache::models::LintKeyInput; + +fn key_sets(keys: &[Vec]) -> String { + let mut sets: Vec = Vec::with_capacity(keys.len()); + for columns in keys { + sets.push(columns.join("\u{1f}")); + } + sets.join("\u{1d}") +} + +/// A digest of the dialect, literal and ranking limits, sorted relation keys and suppressions. +#[must_use] +pub fn lint_key(input: &LintKeyInput) -> String { + let mut relation_keys: Vec<(String, Vec>)> = input.relation_keys.clone(); + relation_keys.sort(); + let relations: String = relation_keys + .iter() + .map(|(relation, keys)| format!("{relation}\u{1e}{}", key_sets(keys))) + .collect::>() + .join("\u{1c}"); + field_digest( + LINT_KEY_KIND, + &[ + &input.dialect, + &input.max_literal_length.to_string(), + &input.max_ranking_order_by.to_string(), + &relations, + if input.allow_suppressions { "1" } else { "0" }, + ], + ) +} diff --git a/crates/sqlbuild-rules/src/rule_cache/main/lookup_rule_cache.rs b/crates/sqlbuild-rules/src/rule_cache/main/lookup_rule_cache.rs new file mode 100644 index 0000000000..2f70facb5c --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/lookup_rule_cache.rs @@ -0,0 +1,19 @@ +//! `lookup_rule_cache`: which paths' stored findings are still valid for their current keys. + +use crate::rule_cache::models::CacheLookup; +use crate::rule_cache::types::CachedEntries; + +/// Hits in key order of `keys`, with every path whose stored key differs or is absent missed. +#[must_use] +pub fn lookup_rule_cache(entries: &CachedEntries, keys: &[(String, String)]) -> CacheLookup { + let mut lookup = CacheLookup::default(); + for (path, key) in keys { + match entries.get(path) { + Some(entry) if entry.identity == *key => { + lookup.hits.push((path.clone(), entry.findings.clone())); + } + _ => lookup.misses.push(path.clone()), + } + } + lookup +} diff --git a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs new file mode 100644 index 0000000000..528348321d --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs @@ -0,0 +1,6 @@ +pub mod file_rule_key; +pub mod lint_key; +pub mod lookup_rule_cache; +pub mod model_rule_key; +pub mod parse_rule_cache; +pub mod rule_cache_text; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs b/crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs new file mode 100644 index 0000000000..99df4d0c9e --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs @@ -0,0 +1,24 @@ +//! `model_rule_key`: the cache key of one model's SQL rule findings. + +use crate::rule_cache::_helpers::digest::field_digest; +use crate::rule_cache::constants::{MODEL_KEY_KIND, NO_CONTRACT}; +use crate::rule_cache::models::ModelKeyInput; + +/// A digest of the lint key, selected codes, authored and compiled SQL and dynamic contract. +#[must_use] +pub fn model_rule_key(lint_key: &str, codes: &[String], model: &ModelKeyInput<'_>) -> String { + let codes: String = codes.join("\0"); + let contract: &str = model.contract.unwrap_or(NO_CONTRACT); + let has_contract: &str = if model.contract.is_some() { "1" } else { "0" }; + field_digest( + MODEL_KEY_KIND, + &[ + lint_key, + &codes, + model.authored_sql, + model.query_sql, + has_contract, + contract, + ], + ) +} diff --git a/crates/sqlbuild-rules/src/rule_cache/main/parse_rule_cache.rs b/crates/sqlbuild-rules/src/rule_cache/main/parse_rule_cache.rs new file mode 100644 index 0000000000..626fa2952d --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/parse_rule_cache.rs @@ -0,0 +1,35 @@ +//! `parse_rule_cache`: the usable entries of a stored SQL rule cache. + +use serde_json::{Map, Value}; + +use crate::constants::NATIVE_BUILD_IDENTITY; +use crate::rule_cache::constants::{ + BUILD_IDENTITY_KEY, CACHE_VERSION, ENTRIES_KEY, FINDINGS_KEY, IDENTITY_KEY, VERSION_KEY, +}; +use crate::rule_cache::models::CachedEntry; +use crate::rule_cache::types::CachedEntries; + +fn entry(value: &Value) -> Option { + let object: &Map = value.as_object()?; + Some(CachedEntry { + identity: object.get(IDENTITY_KEY)?.as_str()?.to_owned(), + findings: object.get(FINDINGS_KEY)?.as_array()?.clone(), + }) +} + +/// Entries of a cache written by this format and native build; anything else reads as empty. +#[must_use] +pub fn parse_rule_cache(text: &str) -> CachedEntries { + let Ok(Value::Object(payload)) = serde_json::from_str::(text) else { + return CachedEntries::new(); + }; + let current: bool = payload.get(VERSION_KEY).and_then(Value::as_str) == Some(CACHE_VERSION) + && payload.get(BUILD_IDENTITY_KEY).and_then(Value::as_str) == Some(NATIVE_BUILD_IDENTITY); + let Some(Value::Object(entries)) = payload.get(ENTRIES_KEY).filter(|_| current) else { + return CachedEntries::new(); + }; + entries + .iter() + .filter_map(|(path, value)| Some((path.clone(), entry(value)?))) + .collect() +} diff --git a/crates/sqlbuild-rules/src/rule_cache/main/rule_cache_text.rs b/crates/sqlbuild-rules/src/rule_cache/main/rule_cache_text.rs new file mode 100644 index 0000000000..0eabbd6065 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/rule_cache_text.rs @@ -0,0 +1,27 @@ +//! `rule_cache_text`: the stored form of the SQL rule cache. + +use serde_json::{Map, Value, json}; + +use crate::constants::NATIVE_BUILD_IDENTITY; +use crate::rule_cache::constants::{ + BUILD_IDENTITY_KEY, CACHE_VERSION, ENTRIES_KEY, FINDINGS_KEY, IDENTITY_KEY, VERSION_KEY, +}; +use crate::rule_cache::types::CachedEntries; + +/// Compact JSON of the format version, native build and every entry by path. +#[must_use] +pub fn rule_cache_text(entries: &CachedEntries) -> String { + let stored: Map = entries + .iter() + .map(|(path, entry)| { + let value = json!({ IDENTITY_KEY: entry.identity, FINDINGS_KEY: entry.findings }); + (path.clone(), value) + }) + .collect(); + json!({ + VERSION_KEY: CACHE_VERSION, + BUILD_IDENTITY_KEY: NATIVE_BUILD_IDENTITY, + ENTRIES_KEY: stored, + }) + .to_string() +} diff --git a/crates/sqlbuild-rules/src/rule_cache/mod.rs b/crates/sqlbuild-rules/src/rule_cache/mod.rs new file mode 100644 index 0000000000..76c842f679 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-rules/src/rule_cache/models.rs b/crates/sqlbuild-rules/src/rule_cache/models.rs new file mode 100644 index 0000000000..accfe26913 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/models.rs @@ -0,0 +1,35 @@ +//! SQL rule cache key inputs, stored entries and lookups. + +/// Everything besides SQL text that changes SQL rule findings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LintKeyInput { + pub dialect: String, + pub max_literal_length: i64, + pub max_ranking_order_by: i64, + /// Each relation and its key column sets, in any order. + pub relation_keys: Vec<(String, Vec>)>, + pub allow_suppressions: bool, +} + +/// One model's SQL rule inputs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelKeyInput<'a> { + pub authored_sql: &'a str, + pub query_sql: &'a str, + /// The model's dynamic column contract in a canonical text form, if it has one. + pub contract: Option<&'a str>, +} + +/// One stored entry: the key it was computed for and its findings as JSON values. +#[derive(Clone, Debug, PartialEq)] +pub struct CachedEntry { + pub identity: String, + pub findings: Vec, +} + +/// The stored findings of each path whose key still matches, and every other path. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CacheLookup { + pub hits: Vec<(String, Vec)>, + pub misses: Vec, +} diff --git a/crates/sqlbuild-rules/src/rule_cache/types.rs b/crates/sqlbuild-rules/src/rule_cache/types.rs new file mode 100644 index 0000000000..d2d2a48893 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/types.rs @@ -0,0 +1,8 @@ +//! The stored entries of one SQL rule cache. + +use std::collections::BTreeMap; + +use crate::rule_cache::models::CachedEntry; + +/// Every usable stored entry by project-relative path. +pub type CachedEntries = BTreeMap; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 744598a126..590b06053b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3081,6 +3081,19 @@ def evaluate_source_freshness_age_policy( ) -> SourceFreshnessAgeStatus | None: ... def redact_diagnostic_text(value: str) -> str: ... def redact_diagnostic_value(*, name: str, value: object) -> object: ... +def sql_rule_lint_key(*, config: LintConfig) -> str: ... +def sql_model_rule_keys( + *, lint_key: str, codes: list[str], models: list[tuple[str, str, str, str | None]] +) -> dict[str, str]: ... +def sql_file_rule_key( + *, lint_key: str, codes: list[str], relative_path: str, contents: str, expanded: bool +) -> str | None: ... +def load_sql_rule_cache(*, path: Path) -> dict[str, tuple[str, str]]: ... +def lookup_sql_rule_cache( + *, entries: dict[str, tuple[str, str]], keys: list[tuple[str, str]] +) -> tuple[dict[str, str], list[str]]: ... +def encode_sql_rule_findings(*, findings: list[Finding]) -> str: ... +def write_sql_rule_cache(*, path: Path, entries: dict[str, tuple[str, str]]) -> None: ... def cursor_parse(*, raw: object, cursor_type: str) -> CursorScalar: ... def cursor_try_parse(*, raw: object, cursor_type: str) -> CursorScalar | None: ... def cursor_render(*, value: CursorScalar) -> str: ... diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index cb50216345..914376ff72 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -2,23 +2,25 @@ from __future__ import annotations -import hashlib import json -import os import time from concurrent.futures import Executor, Future, ThreadPoolExecutor from contextvars import Token from dataclasses import asdict, dataclass, replace from pathlib import Path from threading import Event -from typing import Any -import sqlbuild._native as _native from sqlbuild._native import ( + encode_sql_rule_findings, + load_sql_rule_cache, + lookup_sql_rule_cache, project_with, + sql_file_rule_key, + sql_model_rule_keys, + sql_rule_lint_key, with_duplicate_literal_hints, + write_sql_rule_cache, ) -from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlExpansion, @@ -41,7 +43,6 @@ MAX_LITERAL_LENGTH_THRESHOLD, MAX_RANKING_ORDER_BY_THRESHOLD, SAFE_RULE_FIX_CODES, - TEMPLATE_INTERPOLATION_START, ) from sqlbuild.lint.exceptions import NativeLintError from sqlbuild.lint.main.build_expansion_context import build_expansion_context @@ -80,8 +81,6 @@ SqlExpansionReuse, ) -_SQL_RULE_CACHE_VERSION: str = "sql-rules-v5" -_RULES_BUILD_IDENTITY: str = _native.BUILD_IDENTITY _SQL_RULE_SUPPRESSION_CODE: str = "SQBRSQL000" @@ -189,7 +188,7 @@ def evaluate_rules( sql_rule_inputs: _SqlRuleInputs = _prepare_sql_rule_inputs( rules=native_rules, project=selected_project, - dialect=_lint_identity(lint_config), + dialect=sql_rule_lint_key(config=lint_config), project_dir=resolved_project_dir, collect_files=prepared_sql is None and model_paths is None, discovered_inputs=discovered_inputs, @@ -496,26 +495,21 @@ def _run_sql_rules( if not codes: return _SqlRulesEvaluation(findings=(), cache_hits=0, cache_misses=0) models_by_path: dict[str, CompiledModelView] = _sql_rule_models(project) - bucket: dict[str, dict[str, object]] = ( - _read_sql_rule_cache(project_dir) if cache_enabled else {} - ) + bucket: dict[str, tuple[str, str]] = _read_sql_rule_cache(project_dir) if cache_enabled else {} identities: dict[str, str] = ( prepared_inputs.identities if prepared_inputs is not None else _sql_model_rule_identities( - rules=rules, project=project, dialect=_lint_identity(lint_config) + rules=rules, project=project, dialect=sql_rule_lint_key(config=lint_config) ) ) findings: list[Finding] = [] - misses: set[str] = set() - for path, identity in identities.items(): - cached: tuple[Finding, ...] | None = _cached_sql_findings( - entry=bucket.get(path), identity=identity - ) - if cached is None: - misses.add(path) - else: - findings.extend(cached) + model_hits: dict[str, tuple[Finding, ...]] = _cached_sql_findings( + bucket=bucket, keys=identities + ) + misses: set[str] = set(identities) - set(model_hits) + for path in identities: + findings.extend(model_hits.get(path, ())) selected_paths: set[Path] = {project_dir / path for path in misses} file_identities: dict[str, str] = {} project_files: dict[Path, str] | None = None @@ -536,25 +530,29 @@ def _run_sql_rules( discovered_inputs=discovered_inputs, ) ) + lint_key: str = sql_rule_lint_key(config=lint_config) + file_keys: dict[str, str | None] = { + file_path.relative_to(project_dir).as_posix(): sql_file_rule_key( + lint_key=lint_key, + codes=list(codes), + relative_path=file_path.relative_to(project_dir).as_posix(), + contents=contents, + expanded=file_path in project.sql_expansions, + ) + for file_path, contents in project_files.items() + if file_path not in model_paths + } + file_hits: dict[str, tuple[Finding, ...]] = _cached_sql_findings( + bucket=bucket, + keys={path: key for path, key in file_keys.items() if key is not None}, + ) file_path: Path - contents: str - for file_path, contents in project_files.items(): + for file_path in project_files: if file_path in model_paths: continue relative_path: str = file_path.relative_to(project_dir).as_posix() - file_identity: str | None = _sql_file_rule_identity( - file_path=file_path, - relative_path=relative_path, - contents=contents, - codes=codes, - dialect=_lint_identity(lint_config), - sql_expansions=project.sql_expansions, - ) - cached_file: tuple[Finding, ...] | None = ( - None - if file_identity is None - else _cached_sql_findings(entry=bucket.get(relative_path), identity=file_identity) - ) + file_identity: str | None = file_keys[relative_path] + cached_file: tuple[Finding, ...] | None = file_hits.get(relative_path) if cached_file is None: selected_paths.add(file_path) if file_identity is not None: @@ -617,11 +615,8 @@ def _run_sql_rules( if path in by_path: by_path[path].append(finding) for path, identity in written_identities.items(): - bucket[path] = { - "identity": identity, - "findings": [_finding_cache_payload(item) for item in by_path[path]], - } - _write_sql_rule_cache(project_dir=project_dir, bucket=bucket) + bucket[path] = (identity, encode_sql_rule_findings(findings=by_path[path])) + write_sql_rule_cache(path=_sql_rule_cache_path(project_dir), entries=bucket) return _SqlRulesEvaluation( findings=with_duplicate_literal_hints(findings=tuple(findings), project_dir=project_dir), cache_hits=len(identities) - len(misses), @@ -693,21 +688,6 @@ def sql_lint_config( ) -def _lint_identity(config: LintConfig) -> str: - """Cache identity of everything besides SQL text that changes SQL Rule findings.""" - - return json.dumps( - [ - config.dialect, - config.max_literal_length, - config.max_ranking_order_by, - sorted(config.relation_keys.items()), - config.allow_suppressions, - ], - separators=(",", ":"), - ) - - def _sql_rule_codes(rules: tuple[Rule, ...]) -> tuple[str, ...]: return tuple(rule.code for rule in rules if rule.code.startswith("SQBRSQL")) @@ -726,132 +706,51 @@ def _sql_model_rule_identities( codes: tuple[str, ...] = _sql_rule_codes(rules) if not codes: return {} - return { - path: _sql_rule_identity(model=model, codes=codes, dialect=dialect) + models: list[tuple[str, str, str, str | None]] = [ + (path, model.authored_sql, model.query_sql, _contract_text(model)) for path, model in _sql_rule_models(project).items() - } - - -def _sql_rule_identity(*, model: CompiledModelView, codes: tuple[str, ...], dialect: str) -> str: - digest: Any = hashlib.blake2b(digest_size=32) - digest.update(_SQL_RULE_CACHE_VERSION.encode()) - digest.update(_RULES_BUILD_IDENTITY.encode()) - digest.update(dialect.encode()) - digest.update("\0".join(codes).encode()) - digest.update(model.authored_sql.encode()) - digest.update(model.query_sql.encode()) - if model.dynamic_column_contract is not None: - digest.update( - json.dumps( - asdict(model.dynamic_column_contract), - sort_keys=True, - separators=(",", ":"), - default=str, - ).encode() - ) - return digest.hexdigest() + ] + keys: dict[str, str] = sql_model_rule_keys(lint_key=dialect, codes=list(codes), models=models) + return {path: keys[path] for path, *_ in models} -def _sql_file_rule_identity( - *, - file_path: Path, - relative_path: str, - contents: str, - codes: tuple[str, ...], - dialect: str, - sql_expansions: dict[Path, CompiledSqlExpansion], -) -> str | None: - """Identify non-model SQL whose lint depends only on its text; expansions return None.""" +def _contract_text(model: CompiledModelView) -> str | None: + """Canonical text of the dynamic column contract that SQL rule findings depend on.""" - if ( - MACRO_TOKEN in contents - or TEMPLATE_INTERPOLATION_START in contents - or file_path in sql_expansions - ): + if model.dynamic_column_contract is None: return None - digest: Any = hashlib.blake2b(digest_size=32) - for value in ( - _SQL_RULE_CACHE_VERSION, - "file", - _RULES_BUILD_IDENTITY, - dialect, - "\0".join(codes), - relative_path, - contents, - ): - digest.update(value.encode()) - digest.update(b"\0") - return digest.hexdigest() + return json.dumps( + asdict(model.dynamic_column_contract), + sort_keys=True, + separators=(",", ":"), + default=str, + ) def _sql_rule_cache_path(project_dir: Path) -> Path: return rules_bulk_cache_path(project_dir=project_dir, file_name=SQL_RULES_CACHE_FILE) -def _read_sql_rule_cache(project_dir: Path) -> dict[str, dict[str, object]]: +def _read_sql_rule_cache(project_dir: Path) -> dict[str, tuple[str, str]]: """Return usable entries so incompatible builds cannot suppress early lint.""" - path: Path = _sql_rule_cache_path(project_dir) - try: - payload: object = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, json.JSONDecodeError): - return {} - if ( - not isinstance(payload, dict) - or payload.get("version") != _SQL_RULE_CACHE_VERSION - or payload.get("build_identity") != _RULES_BUILD_IDENTITY - ): - return {} - entries: object = payload.get("entries") - if not isinstance(entries, dict): - return {} - return {str(key): value for key, value in entries.items() if isinstance(value, dict)} + return load_sql_rule_cache(path=_sql_rule_cache_path(project_dir)) def _cached_sql_findings( - *, entry: dict[str, object] | None, identity: str -) -> tuple[Finding, ...] | None: - if entry is None or entry.get("identity") != identity: - return None - values: object = entry.get("findings") - if not isinstance(values, list): - return None - try: - return tuple(decode_rule_finding(value) for value in values) - except (KeyError, RulesError, TypeError, ValueError): - return None - - -def _finding_cache_payload(finding: Finding) -> dict[str, object]: - return { - "unevaluated": finding.unevaluated, - "code": finding.code, - "path": finding.path.as_posix(), - "line": finding.line, - "column": finding.column, - "message": finding.message, - "remediation": finding.remediation, - "fixable": finding.fixable, - } - - -def _write_sql_rule_cache(*, project_dir: Path, bucket: dict[str, dict[str, object]]) -> None: - path: Path = _sql_rule_cache_path(project_dir) - path.parent.mkdir(parents=True, exist_ok=True) - temporary: Path = path.with_suffix(f".tmp-{os.getpid()}") - temporary.write_text( - json.dumps( - { - "version": _SQL_RULE_CACHE_VERSION, - "build_identity": _RULES_BUILD_IDENTITY, - "entries": bucket, - }, - sort_keys=True, - separators=(",", ":"), - ), - encoding="utf-8", - ) - temporary.replace(path) + *, bucket: dict[str, tuple[str, str]], keys: dict[str, str] +) -> dict[str, tuple[Finding, ...]]: + """Decoded stored findings of each path whose key still matches; undecodable ones miss.""" + + hits: dict[str, str] + hits, _ = lookup_sql_rule_cache(entries=bucket, keys=list(keys.items())) + decoded: dict[str, tuple[Finding, ...]] = {} + for path, payload in hits.items(): + try: + decoded[path] = tuple(decode_rule_finding(value) for value in json.loads(payload)) + except (KeyError, RulesError, TypeError, ValueError): + continue + return decoded def _selected_model_paths( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 7aa8240085..791f88a676 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1080,3 +1080,12 @@ class BuildManifestTestCase: description: str model_count: int expected_test_type: str + + +@dataclass(frozen=True) +class SqlRuleCacheSequenceTestCase: + """Compiles after each `(project toml, model SQL)` step and the cache counts each reports.""" + + description: str + steps: tuple[tuple[str, str], ...] + expected_hits_and_misses: tuple[tuple[int, int], ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d9efef4b99..fc513adc9c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -32,6 +32,7 @@ NumericRangeDecisionIntegrationTestCase, RulePassIntegrationTestCase, RulesIntegrationTestCase, + SqlRuleCacheSequenceTestCase, TypedContractRuleIntegrationTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( @@ -41,6 +42,16 @@ write_project_files, ) +_CACHE_RULES_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n' +_CACHE_THRESHOLD_TOML: str = _CACHE_RULES_TOML + "thresholds = { max_literal_length = 7 }\n" +_CACHE_POLICY_TOML: str = _CACHE_THRESHOLD_TOML + "allow_exceptions = false\n" +_CACHE_SELECTION_TOML: str = _CACHE_POLICY_TOML.replace( + '"SQBRSQL004"', '"SQBRSQL004", "SQBRSQL001"' +) +_CACHE_MODEL_SQL: str = 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n' +_CACHE_EDITED_SQL: str = _CACHE_MODEL_SQL.replace("LIMIT 1", "LIMIT 2") + + _EARLY_LINT_STOP_TIMEOUT_S: float = 30.0 _EARLY_LINT_FINDINGS: tuple[tuple[str, str, int | None, int | None], ...] = ( ("SQBRSQL005", "models/order_cents.sql", 2, 116), @@ -671,7 +682,6 @@ def test_given_sql_rule_cache_from_another_native_build_when_compiling_then_rule test_case: RulesIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', @@ -687,11 +697,10 @@ def test_given_sql_rule_cache_from_another_native_build_when_compiling_then_rule cold_exit: int = main(command) cold: dict[str, object] = json.loads(capsys.readouterr().out) - monkeypatch.setattr( - rules_module, - "_RULES_BUILD_IDENTITY", - f"{rules_module._RULES_BUILD_IDENTITY}-rebuilt", - ) + cache_path: Path = rules_module._sql_rule_cache_path(tmp_path) + stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) + stored["build_identity"] = f"{stored['build_identity']}-rebuilt" + cache_path.write_text(json.dumps(stored), encoding="utf-8") rebuilt_exit: int = main(command) rebuilt: dict[str, object] = json.loads(capsys.readouterr().out) warm_exit: int = main(command) @@ -976,14 +985,15 @@ def test_given_previous_build_rule_cache_when_compiling_then_lint_overlaps_with_ test_case: EarlyLintInputsTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: write_project_files(project_dir=tmp_path, files=EARLY_LINT_PROJECT_FILES) - with monkeypatch.context() as previous_build: - previous_build.setattr(rules_module, "_RULES_BUILD_IDENTITY", "previous-build") - previous_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( - project_dir=tmp_path, capsys=capsys - ) + previous_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( + project_dir=tmp_path, capsys=capsys + ) + cache_path: Path = rules_module._sql_rule_cache_path(tmp_path) + stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) + stored["build_identity"] = "previous-build" + cache_path.write_text(json.dumps(stored), encoding="utf-8") upgraded_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( project_dir=tmp_path, capsys=capsys ) @@ -2788,3 +2798,52 @@ def required_prefix(*, model: Model, ctx: RuleContext) -> list[Finding]: assert exit_code == test_case.expected_exit_code assert payload["diagnostics"] == [] + + +@pytest.mark.parametrize( + "test_case", + [ + SqlRuleCacheSequenceTestCase( + description="cold misses, warm hits and each SQL rule input edit misses", + steps=( + (_CACHE_RULES_TOML, _CACHE_MODEL_SQL), + (_CACHE_RULES_TOML, _CACHE_MODEL_SQL), + (_CACHE_RULES_TOML, _CACHE_EDITED_SQL), + (_CACHE_RULES_TOML, _CACHE_EDITED_SQL), + (_CACHE_THRESHOLD_TOML, _CACHE_EDITED_SQL), + (_CACHE_POLICY_TOML, _CACHE_EDITED_SQL), + (_CACHE_SELECTION_TOML, _CACHE_EDITED_SQL), + (_CACHE_SELECTION_TOML, _CACHE_EDITED_SQL), + ), + expected_hits_and_misses=( + (0, 1), + (1, 0), + (0, 1), + (1, 0), + (0, 1), + (0, 1), + (0, 1), + (1, 0), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_sql_rule_cache_when_inputs_change_then_only_unchanged_runs_hit( + test_case: SqlRuleCacheSequenceTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + observed: list[tuple[int, int]] = [] + for project_toml, model_sql in test_case.steps: + (tmp_path / "sqlbuild_project.toml").write_text(project_toml, encoding="utf-8") + model.write_text(model_sql, encoding="utf-8") + _ = main(["--project-dir", str(tmp_path), "compile", "--json"]) + timings: dict[str, int] = cast( + dict[str, int], json.loads(capsys.readouterr().out)["compile_timings"] + ) + observed.append((timings["rule_cache_hits"], timings["rule_cache_misses"])) + + assert tuple(observed) == test_case.expected_hits_and_misses From 7445931af8f8717dce779ee9fc363dd7ec61a6ea Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 18:07:31 +0100 Subject: [PATCH 17/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 6cbd9ef0f8..14458477ce 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 85701 +remaining_lines = 84966 [[stays]] path = "executor" From 222510dc8183ba06cb0f451e052d1b93293eaf58 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:10:44 +0100 Subject: [PATCH 18/37] fix: drop command binding imports from the main registration --- crates/sqlbuild-python/src/bindings/main/register.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 4e7e172b80..3037c000d6 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,12 +5,11 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, - clone, command_runtime, compile_attachment, compile_command, compile_driver, compile_outputs, - compiled_project, contracts, cost_report, cursor, dag_artifact, debug_command, diagnostics, - discovery, fingerprints, freshness, frontier, graph, janitor, lineage, macro_bridge, manifest, - model_config, plan_output, presentation_text, project_assembly, refactor, references, render, - result_output, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, - state_rows, type_system, workspace_commands, + compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, + contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, + lineage, macro_bridge, manifest, model_config, plan_output, project_assembly, refactor, + references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, + state_rows, type_system, }; use crate::bindings::models; From 00c7dadbfa0c9c80b271ececb81a1e7ba4a38483 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:29:44 +0100 Subject: [PATCH 19/37] refactor(cli): run sqb check natively --- crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- .../check_command/_helpers/json_payload.rs | 58 ++ .../src/check_command/_helpers/mod.rs | 1 + .../src/check_command/constants.rs | 8 + .../sqlbuild-core/src/check_command/errors.rs | 8 + .../src/check_command/main/check_exit_code.rs | 13 + .../check_command/main/check_group_header.rs | 7 + .../check_command/main/check_group_label.rs | 24 + .../src/check_command/main/check_json.rs | 50 ++ .../src/check_command/main/mod.rs | 7 + .../check_command/main/render_check_row.rs | 33 + .../main/render_check_summary.rs | 31 + .../main/validate_check_selection.rs | 20 + crates/sqlbuild-core/src/check_command/mod.rs | 10 + .../sqlbuild-core/src/check_command/models.rs | 61 ++ .../sqlbuild-core/src/check_command/tests.rs | 6 + .../src/check_command/tests/helpers.rs | 24 + .../check_command/tests/test_check_command.rs | 190 +++++ .../src/check_command/tests/test_types.rs | 28 + crates/sqlbuild-core/src/lib.rs | 1 + .../bindings/_helpers/check/check_command.rs | 754 ++++++++++++++++++ .../bindings/_helpers/check/check_results.rs | 230 ++++++ .../src/bindings/_helpers/check/mod.rs | 3 + .../bindings/_helpers/check/registration.rs | 10 + .../src/bindings/_helpers/cli/registration.rs | 5 +- .../src/bindings/_helpers/mod.rs | 1 + src/sqlbuild/_native.pyi | 12 + .../_helpers/build_execution/checks.py | 3 +- .../cli/commands/_helpers/check/core.py | 237 +----- .../cli/commands/_helpers/check/execution.py | 272 ------- .../cli/commands/_helpers/check/invocation.py | 14 - .../cli/commands/_helpers/check/outputs.py | 73 -- .../cli/commands/_helpers/check/planning.py | 104 --- .../cli/commands/main/execution/_check.py | 66 -- src/sqlbuild/cli/commands/models.py | 18 +- .../{relation_plan.py => _relation_plan.py} | 0 .../runtime/test_command_invocation.py | 10 +- .../commands/_helpers/check/test_outputs.py | 121 +-- .../commands/_helpers/check/test_planning.py | 30 - .../_helpers/test_python_checks.py | 2 +- 40 files changed, 1608 insertions(+), 941 deletions(-) create mode 100644 crates/sqlbuild-core/src/check_command/_helpers/json_payload.rs create mode 100644 crates/sqlbuild-core/src/check_command/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/check_command/constants.rs create mode 100644 crates/sqlbuild-core/src/check_command/errors.rs create mode 100644 crates/sqlbuild-core/src/check_command/main/check_exit_code.rs create mode 100644 crates/sqlbuild-core/src/check_command/main/check_group_header.rs create mode 100644 crates/sqlbuild-core/src/check_command/main/check_group_label.rs create mode 100644 crates/sqlbuild-core/src/check_command/main/check_json.rs create mode 100644 crates/sqlbuild-core/src/check_command/main/mod.rs create mode 100644 crates/sqlbuild-core/src/check_command/main/render_check_row.rs create mode 100644 crates/sqlbuild-core/src/check_command/main/render_check_summary.rs create mode 100644 crates/sqlbuild-core/src/check_command/main/validate_check_selection.rs create mode 100644 crates/sqlbuild-core/src/check_command/mod.rs create mode 100644 crates/sqlbuild-core/src/check_command/models.rs create mode 100644 crates/sqlbuild-core/src/check_command/tests.rs create mode 100644 crates/sqlbuild-core/src/check_command/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/check_command/tests/test_check_command.rs create mode 100644 crates/sqlbuild-core/src/check_command/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/check/execution.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/check/invocation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/check/outputs.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/check/planning.py delete mode 100644 src/sqlbuild/cli/commands/main/execution/_check.py rename src/sqlbuild/compiler/planner/main/commands/{relation_plan.py => _relation_plan.py} (100%) diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index e7d4191d10..f2398f3505 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -262,9 +262,9 @@ const CHECK_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const CHECK_ROUTE: Route = routed( "run_check", - ("sqlbuild.cli.commands.main.execution._check", "run_check"), + ("sqlbuild._native", "run_check_command"), &CHECK_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const AUDIT_FIELDS: [Field; 11] = [ diff --git a/crates/sqlbuild-core/src/check_command/_helpers/json_payload.rs b/crates/sqlbuild-core/src/check_command/_helpers/json_payload.rs new file mode 100644 index 0000000000..b0cc9bd0bd --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/_helpers/json_payload.rs @@ -0,0 +1,58 @@ +//! JSON payload pieces of check output. + +use crate::check_command::models::{CheckResultText, CheckStatus}; +use crate::json::models::{JsonInteger, JsonValue}; + +/// A JSON string, or null. +pub(crate) fn text(value: Option<&str>) -> JsonValue { + value.map_or(JsonValue::Null, |value| JsonValue::String(value.to_owned())) +} + +/// A JSON integer. +pub(crate) fn integer(value: usize) -> JsonValue { + JsonValue::Integer(JsonInteger::from(i64::try_from(value).unwrap_or(i64::MAX))) +} + +/// A JSON object with entries in order. +pub(crate) fn object(entries: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + entries + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ) +} + +/// The JSON entry of one check. +pub(crate) fn check(result: &CheckResultText) -> JsonValue { + let warnings: JsonValue = if result.warning_messages.is_empty() { + JsonValue::Null + } else { + JsonValue::Array( + result + .warning_messages + .iter() + .map(|warning| JsonValue::String(warning.clone())) + .collect(), + ) + }; + object(vec![ + ("kind", text(Some("python_check"))), + ("name", text(Some(&result.name))), + ("display_name", text(Some(&result.name))), + ( + "check_id", + text(Some(&format!("python_check:{}", result.name))), + ), + ("status", text(Some(result.status.value()))), + ( + "passed", + JsonValue::Bool(result.status == CheckStatus::Pass), + ), + ("severity", text(Some(&result.severity))), + ("message", text(result.message.as_deref())), + ("error_message", text(result.error_message.as_deref())), + ("metadata", result.metadata.clone()), + ("warnings", warnings), + ]) +} diff --git a/crates/sqlbuild-core/src/check_command/_helpers/mod.rs b/crates/sqlbuild-core/src/check_command/_helpers/mod.rs new file mode 100644 index 0000000000..ac8e608de3 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod json_payload; diff --git a/crates/sqlbuild-core/src/check_command/constants.rs b/crates/sqlbuild-core/src/check_command/constants.rs new file mode 100644 index 0000000000..f788cb85bd --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/constants.rs @@ -0,0 +1,8 @@ +//! Check output labels and widths. + +pub(crate) const KIND_LABEL: &str = "check"; +pub(crate) const KIND_WIDTH: usize = 10; +pub(crate) const NAME_WIDTH: usize = 50; +pub(crate) const MULTI_DEPENDENCY_GROUP: &str = "multi-dependency"; +pub(crate) const UNGROUPED: &str = "ungrouped"; +pub(crate) const SELECTION_CODE: &str = "C682"; diff --git a/crates/sqlbuild-core/src/check_command/errors.rs b/crates/sqlbuild-core/src/check_command/errors.rs new file mode 100644 index 0000000000..f166404d9c --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/errors.rs @@ -0,0 +1,8 @@ +//! Check selection errors. + +/// A selection that resolved to Python nodes other than checks. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CheckSelectionError { + pub code: &'static str, + pub message: String, +} diff --git a/crates/sqlbuild-core/src/check_command/main/check_exit_code.rs b/crates/sqlbuild-core/src/check_command/main/check_exit_code.rs new file mode 100644 index 0000000000..21e100fad9 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/check_exit_code.rs @@ -0,0 +1,13 @@ +//! The exit code of `sqb check`. + +use crate::check_command::models::{CheckResultText, CheckStatus}; + +/// 1 when any check failed, else 0. +#[must_use] +pub fn check_exit_code(results: &[CheckResultText]) -> i32 { + i32::from( + results + .iter() + .any(|result| result.status == CheckStatus::Fail), + ) +} diff --git a/crates/sqlbuild-core/src/check_command/main/check_group_header.rs b/crates/sqlbuild-core/src/check_command/main/check_group_header.rs new file mode 100644 index 0000000000..92bd0eaea0 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/check_group_header.rs @@ -0,0 +1,7 @@ +//! The heading of one check group. + +/// `Python checks: ` and a blank line. +#[must_use] +pub fn check_group_header(group: &str) -> String { + format!("Python checks: {group}\n\n") +} diff --git a/crates/sqlbuild-core/src/check_command/main/check_group_label.rs b/crates/sqlbuild-core/src/check_command/main/check_group_label.rs new file mode 100644 index 0000000000..22bb5d89c2 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/check_group_label.rs @@ -0,0 +1,24 @@ +//! The output group of one check. + +use crate::check_command::constants::{MULTI_DEPENDENCY_GROUP, UNGROUPED}; +use crate::check_command::models::CheckGrouping; + +/// The single dependency's name, else the declared group, else `multi-dependency` or `ungrouped`. +#[must_use] +pub fn check_group_label(grouping: &CheckGrouping) -> String { + if let Some(dependencies) = &grouping.dependencies { + if let [only] = dependencies.as_slice() { + return only.clone(); + } + if dependencies.len() > 1 { + return grouping + .group + .clone() + .unwrap_or_else(|| MULTI_DEPENDENCY_GROUP.to_owned()); + } + } + grouping + .group + .clone() + .unwrap_or_else(|| UNGROUPED.to_owned()) +} diff --git a/crates/sqlbuild-core/src/check_command/main/check_json.rs b/crates/sqlbuild-core/src/check_command/main/check_json.rs new file mode 100644 index 0000000000..cbf7b77842 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/check_json.rs @@ -0,0 +1,50 @@ +//! The JSON payload of `sqb check`. + +use crate::check_command::_helpers::json_payload::{check, integer, object, text}; +use crate::check_command::models::{CheckResultText, CheckStatus}; +use crate::json::errors::JsonEmitError; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// Status, counts and checks as `json.dumps(..., indent=2)` with a trailing newline. +pub fn check_json(results: &[CheckResultText]) -> Result { + let count = |status: CheckStatus| { + integer( + results + .iter() + .filter(|result| result.status == status) + .count(), + ) + }; + let failed: bool = results + .iter() + .any(|result| result.status == CheckStatus::Fail); + let payload: JsonValue = object(vec![ + ("version", integer(1)), + ("command", text(Some("check"))), + ( + "status", + text(Some(if failed { "failed" } else { "success" })), + ), + ( + "summary", + object(vec![ + ("pass_count", count(CheckStatus::Pass)), + ("warn_count", count(CheckStatus::Warn)), + ("fail_count", count(CheckStatus::Fail)), + ("total_count", integer(results.len())), + ]), + ), + ( + "checks", + JsonValue::Array(results.iter().map(check).collect()), + ), + ]); + Ok(format!( + "{}\n", + dumps( + &payload, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))) + )? + )) +} diff --git a/crates/sqlbuild-core/src/check_command/main/mod.rs b/crates/sqlbuild-core/src/check_command/main/mod.rs new file mode 100644 index 0000000000..1b40c2cabb --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/mod.rs @@ -0,0 +1,7 @@ +pub mod check_exit_code; +pub mod check_group_header; +pub mod check_group_label; +pub mod check_json; +pub mod render_check_row; +pub mod render_check_summary; +pub mod validate_check_selection; diff --git a/crates/sqlbuild-core/src/check_command/main/render_check_row.rs b/crates/sqlbuild-core/src/check_command/main/render_check_row.rs new file mode 100644 index 0000000000..b3b0241a8c --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/render_check_row.rs @@ -0,0 +1,33 @@ +//! One human check result row. + +use crate::check_command::constants::{KIND_LABEL, KIND_WIDTH, NAME_WIDTH}; +use crate::check_command::models::CheckRow; +use crate::cli_text::models::CliStyle; + +/// Kind, padded name, status, duration and detail, then each warning on its own line. +#[must_use] +pub fn render_check_row(check: &CheckRow, style: CliStyle) -> String { + let result = &check.result; + let padding: String = " ".repeat(NAME_WIDTH.saturating_sub(result.name.chars().count())); + let mut text: String = format!( + " {KIND_LABEL: = result + .error_message + .as_ref() + .filter(|error| !error.is_empty()) + .or(result.message.as_ref()); + if let Some(detail) = detail.filter(|detail| !detail.is_empty()) { + text.push_str(&format!(" {detail}")); + } + text.push('\n'); + for warning in &result.warning_messages { + text.push_str(&format!(" {}\n", style.warning(warning))); + } + text +} diff --git a/crates/sqlbuild-core/src/check_command/main/render_check_summary.rs b/crates/sqlbuild-core/src/check_command/main/render_check_summary.rs new file mode 100644 index 0000000000..5e9d323311 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/render_check_summary.rs @@ -0,0 +1,31 @@ +//! The check summary footer. + +use crate::check_command::models::{CheckResultText, CheckStatus}; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, SummaryCount}; + +fn count(results: &[CheckResultText], status: Option) -> i64 { + let matched: usize = results + .iter() + .filter(|result| status.is_none_or(|status| result.status == status)) + .count(); + i64::try_from(matched).unwrap_or(i64::MAX) +} + +/// A blank line and the PASS, WARN, FAIL and TOTAL counts. +#[must_use] +pub fn render_check_summary(results: &[CheckResultText], style: CliStyle) -> String { + let counts: Vec = [ + ("PASS", Some(CheckStatus::Pass)), + ("WARN", Some(CheckStatus::Warn)), + ("FAIL", Some(CheckStatus::Fail)), + ("TOTAL", None), + ] + .into_iter() + .map(|(label, status)| SummaryCount { + label: label.to_owned(), + value: count(results, status), + }) + .collect(); + format!("\n{}\n", format_summary_footer(style, &counts, None)) +} diff --git a/crates/sqlbuild-core/src/check_command/main/validate_check_selection.rs b/crates/sqlbuild-core/src/check_command/main/validate_check_selection.rs new file mode 100644 index 0000000000..3a47bea587 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/validate_check_selection.rs @@ -0,0 +1,20 @@ +//! Refuse a check selection that resolved to Python nodes other than checks. + +use crate::check_command::constants::SELECTION_CODE; +use crate::check_command::errors::CheckSelectionError; + +/// C682 naming the sorted non-check selections, when there are any. +pub fn validate_check_selection(non_check_names: &[String]) -> Result<(), CheckSelectionError> { + if non_check_names.is_empty() { + return Ok(()); + } + let mut names: Vec<&str> = non_check_names.iter().map(String::as_str).collect(); + names.sort_unstable(); + Err(CheckSelectionError { + code: SELECTION_CODE, + message: format!( + "sqb check selectors must resolve only to Python checks; non-check selections: {}", + names.join(", ") + ), + }) +} diff --git a/crates/sqlbuild-core/src/check_command/mod.rs b/crates/sqlbuild-core/src/check_command/mod.rs new file mode 100644 index 0000000000..85a89a753f --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/mod.rs @@ -0,0 +1,10 @@ +//! `sqb check`: check selection errors, grouped result rows, JSON payload and the exit code. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/check_command/models.rs b/crates/sqlbuild-core/src/check_command/models.rs new file mode 100644 index 0000000000..6214d0a615 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/models.rs @@ -0,0 +1,61 @@ +//! Python check results as the command renders them. + +use crate::json::models::JsonValue; + +/// How one check ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CheckStatus { + Pass, + Warn, + Fail, +} + +impl CheckStatus { + /// The label rows and the summary use. + #[must_use] + pub fn label(self) -> &'static str { + match self { + Self::Pass => "PASS", + Self::Warn => "WARN", + Self::Fail => "FAIL", + } + } + + /// The status JSON output spells. + #[must_use] + pub fn value(self) -> &'static str { + match self { + Self::Pass => "pass", + Self::Warn => "warn", + Self::Fail => "fail", + } + } +} + +/// One executed Python check. +#[derive(Clone, Debug, PartialEq)] +pub struct CheckResultText { + pub name: String, + pub status: CheckStatus, + /// The `CheckSeverity` value. + pub severity: String, + pub message: Option, + pub error_message: Option, + pub metadata: JsonValue, + pub warning_messages: Vec, +} + +/// What decides a check's output group: its dependencies and declared group. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CheckGrouping { + /// Python dependency names, then SQL dependency names; `None` without a Python graph. + pub dependencies: Option>, + pub group: Option, +} + +/// One rendered row: the result and the duration the progress projector recorded. +#[derive(Clone, Debug, PartialEq)] +pub struct CheckRow { + pub result: CheckResultText, + pub duration_ms: Option, +} diff --git a/crates/sqlbuild-core/src/check_command/tests.rs b/crates/sqlbuild-core/src/check_command/tests.rs new file mode 100644 index 0000000000..3665959f9c --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_check_command.rs"] +mod check_command; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/check_command/tests/helpers.rs b/crates/sqlbuild-core/src/check_command/tests/helpers.rs new file mode 100644 index 0000000000..892194ecf9 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/tests/helpers.rs @@ -0,0 +1,24 @@ +use crate::check_command::models::{CheckGrouping, CheckResultText, CheckStatus}; +use crate::json::models::JsonValue; + +/// A check result with no message, metadata or warnings. +pub(super) fn result(name: &str, status: CheckStatus) -> CheckResultText { + CheckResultText { + name: name.to_owned(), + status, + severity: "error".to_owned(), + message: None, + error_message: None, + metadata: JsonValue::Object(Vec::new()), + warning_messages: Vec::new(), + } +} + +/// A grouping with dependencies and a declared group. +pub(super) fn grouping(dependencies: Option<&[&str]>, group: Option<&str>) -> CheckGrouping { + CheckGrouping { + dependencies: dependencies + .map(|names| names.iter().map(|name| (*name).to_owned()).collect()), + group: group.map(str::to_owned), + } +} diff --git a/crates/sqlbuild-core/src/check_command/tests/test_check_command.rs b/crates/sqlbuild-core/src/check_command/tests/test_check_command.rs new file mode 100644 index 0000000000..f28fd941a5 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/tests/test_check_command.rs @@ -0,0 +1,190 @@ +use crate::check_command::main::check_exit_code::check_exit_code; +use crate::check_command::main::check_group_label::check_group_label; +use crate::check_command::main::check_json::check_json; +use crate::check_command::main::render_check_row::render_check_row; +use crate::check_command::main::render_check_summary::render_check_summary; +use crate::check_command::main::validate_check_selection::validate_check_selection; +use crate::check_command::models::{CheckResultText, CheckRow, CheckStatus}; +use crate::check_command::tests::helpers::{grouping, result}; +use crate::check_command::tests::test_types::{ + GroupTestCase, RowTestCase, SelectionTestCase, SummaryTestCase, +}; +use crate::cli_text::models::CliStyle; +use crate::json::models::{JsonInteger, JsonValue}; + +#[test] +fn given_dependencies_when_grouping_then_labels_by_single_dependency_or_group() { + let test_cases = [ + GroupTestCase { + description: "one dependency names the group", + grouping: grouping(Some(&["export_orders"]), Some("declared")), + expected_label: "export_orders", + }, + GroupTestCase { + description: "several dependencies use the declared group", + grouping: grouping(Some(&["a", "b"]), Some("declared")), + expected_label: "declared", + }, + GroupTestCase { + description: "several dependencies without a group are multi-dependency", + grouping: grouping(Some(&["a", "b"]), None), + expected_label: "multi-dependency", + }, + GroupTestCase { + description: "no graph and no group is ungrouped", + grouping: grouping(None, None), + expected_label: "ungrouped", + }, + ]; + + for test_case in test_cases { + assert_eq!( + check_group_label(&test_case.grouping), + test_case.expected_label, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_row_when_rendering_then_pads_name_and_shows_detail() { + let test_cases = [ + RowTestCase { + description: "an error message wins over the message and warnings follow", + row: CheckRow { + result: CheckResultText { + message: Some("ignored".to_owned()), + error_message: Some("boom".to_owned()), + warning_messages: vec!["careful".to_owned()], + ..result("orders", CheckStatus::Fail) + }, + duration_ms: Some(125.0), + }, + use_color: false, + expected_text: " check orders FAIL 0.12s boom\n careful\n", + }, + RowTestCase { + description: "colour styles the status and an empty error falls back to the message", + row: CheckRow { + result: CheckResultText { + message: Some("fine".to_owned()), + error_message: Some(String::new()), + ..result("orders", CheckStatus::Pass) + }, + duration_ms: None, + }, + use_color: true, + expected_text: " check orders \u{1b}[32mPASS\u{1b}[0m fine\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_check_row( + &test_case.row, + CliStyle { + use_color: test_case.use_color + } + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_summarizing_then_counts_serializes_and_exits() { + let test_cases = [ + SummaryTestCase { + description: "a warning does not fail the run", + results: vec![ + CheckResultText { + metadata: JsonValue::Object(vec![( + "rows".to_owned(), + JsonValue::Integer(JsonInteger::from(3_i64)), + )]), + ..result("a", CheckStatus::Pass) + }, + CheckResultText { + warning_messages: vec!["w".to_owned()], + severity: "warn".to_owned(), + ..result("b", CheckStatus::Warn) + }, + ], + expected_summary: "\nPASS=1 WARN=1 FAIL=0 TOTAL=2\n", + expected_json: "{\n \"version\": 1,\n \"command\": \"check\",\n \"status\": \"success\",\n \ + \"summary\": {\n \"pass_count\": 1,\n \"warn_count\": 1,\n \"fail_count\": 0,\n \ + \"total_count\": 2\n },\n \"checks\": [\n {\n \"kind\": \"python_check\",\n \ + \"name\": \"a\",\n \"display_name\": \"a\",\n \"check_id\": \"python_check:a\",\n \ + \"status\": \"pass\",\n \"passed\": true,\n \"severity\": \"error\",\n \ + \"message\": null,\n \"error_message\": null,\n \"metadata\": {\n \ + \"rows\": 3\n },\n \"warnings\": null\n },\n {\n \ + \"kind\": \"python_check\",\n \"name\": \"b\",\n \"display_name\": \"b\",\n \ + \"check_id\": \"python_check:b\",\n \"status\": \"warn\",\n \"passed\": false,\n \ + \"severity\": \"warn\",\n \"message\": null,\n \"error_message\": null,\n \ + \"metadata\": {},\n \"warnings\": [\n \"w\"\n ]\n }\n ]\n}\n", + expected_exit_code: 0, + }, + SummaryTestCase { + description: "a failure fails the run", + results: vec![result("c", CheckStatus::Fail)], + expected_summary: "\nPASS=0 WARN=0 FAIL=1 TOTAL=1\n", + expected_json: "{\n \"version\": 1,\n \"command\": \"check\",\n \"status\": \"failed\",\n \ + \"summary\": {\n \"pass_count\": 0,\n \"warn_count\": 0,\n \"fail_count\": 1,\n \ + \"total_count\": 1\n },\n \"checks\": [\n {\n \"kind\": \"python_check\",\n \ + \"name\": \"c\",\n \"display_name\": \"c\",\n \"check_id\": \"python_check:c\",\n \ + \"status\": \"fail\",\n \"passed\": false,\n \"severity\": \"error\",\n \ + \"message\": null,\n \"error_message\": null,\n \"metadata\": {},\n \ + \"warnings\": null\n }\n ]\n}\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + render_check_summary(&test_case.results, CliStyle { use_color: false }), + check_json(&test_case.results).ok(), + check_exit_code(&test_case.results), + ), + ( + test_case.expected_summary.to_owned(), + Some(test_case.expected_json.to_owned()), + test_case.expected_exit_code, + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selected_names_when_validating_then_rejects_non_checks() { + let test_cases = [ + SelectionTestCase { + description: "only checks pass", + non_checks: Vec::new(), + expected_message: None, + }, + SelectionTestCase { + description: "non-check names are listed sorted", + non_checks: vec!["task_b".to_owned(), "asset_a".to_owned()], + expected_message: Some( + "sqb check selectors must resolve only to Python checks; non-check selections: asset_a, task_b", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + validate_check_selection(&test_case.non_checks) + .err() + .map(|error| error.message), + test_case.expected_message.map(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/check_command/tests/test_types.rs b/crates/sqlbuild-core/src/check_command/tests/test_types.rs new file mode 100644 index 0000000000..967c8d64ab --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/tests/test_types.rs @@ -0,0 +1,28 @@ +use crate::check_command::models::{CheckGrouping, CheckResultText, CheckRow}; + +pub(super) struct GroupTestCase { + pub(super) description: &'static str, + pub(super) grouping: CheckGrouping, + pub(super) expected_label: &'static str, +} + +pub(super) struct RowTestCase { + pub(super) description: &'static str, + pub(super) row: CheckRow, + pub(super) use_color: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct SummaryTestCase { + pub(super) description: &'static str, + pub(super) results: Vec, + pub(super) expected_summary: &'static str, + pub(super) expected_json: &'static str, + pub(super) expected_exit_code: i32, +} + +pub(super) struct SelectionTestCase { + pub(super) description: &'static str, + pub(super) non_checks: Vec, + pub(super) expected_message: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index f63645fa4f..c5b22fb902 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -2,6 +2,7 @@ #![forbid(unsafe_code)] +pub mod check_command; pub mod cli_text; pub mod clone_text; pub mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs new file mode 100644 index 0000000000..a53c860a6b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -0,0 +1,754 @@ +//! `sqb check`: run the selected Python checks after their Python dependencies. + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyInt, PyString, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::check_command::errors::CheckSelectionError; +use sqlbuild_core::check_command::main::check_exit_code::check_exit_code; +use sqlbuild_core::check_command::main::render_check_summary::render_check_summary; +use sqlbuild_core::check_command::main::validate_check_selection::validate_check_selection; +use sqlbuild_core::check_command::models::CheckResultText; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::check::check_results::{ + check_result_text, results_json, write_results, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; +const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main.static_command"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const RELATION_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relation_plan"; +const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; +const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main.graph"; +const PYTHON_SELECTORS_MODULE: &str = "sqlbuild.compiler.python_nodes.main.selectors"; +const PYTHON_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main.run_lifecycle"; +const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; +const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; +const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; +const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main.checks"; +const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main.ingress"; +const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; +const RUNTIME_TARGET_MODULE: &str = + "sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target"; +const GRAPH_OPERATOR_CODE: &str = "C681"; +const GRAPH_OPERATOR_MESSAGE: &str = "sqb check selectors do not support graph operators; \ + select checks directly without '+', '~', or path-between expansion"; +const EXECUTION_COMMAND: &str = "sqb check"; + +/// The compiled pipeline and everything checks run with. +struct Prepared<'py> { + pipeline: Bound<'py, PyAny>, + python_graph: Bound<'py, PyAny>, + check_functions: Bound<'py, PyAny>, + lifecycle_plan: Bound<'py, PyAny>, + relation_targets: Bound<'py, PyAny>, + relation_refs: Bound<'py, PyAny>, + defaults: (Bound<'py, PyAny>, Bound<'py, PyAny>), + project_relations: Bound<'py, PyAny>, +} + +fn user_error(py: Python<'_>, (code, message): (&str, &str)) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", code)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn selection_error(py: Python<'_>, error: &CheckSelectionError) -> PyErr { + user_error(py, (error.code, &error.message)) +} + +fn empty_tuple(py: Python<'_>) -> Bound<'_, PyAny> { + PyTuple::empty(py).into_any() +} + +/// Refuse `+`, `~` and path selectors, splitting selectors as Python's `split` does. +fn refuse_graph_selectors(py: Python<'_>, selectors: &[Bound<'_, PyAny>]) -> PyResult<()> { + let parse: Bound<'_, PyAny> = imported(py, SELECTOR_PARSE_MODULE, "parse_project_selector")?; + let path_selector: Bound<'_, PyAny> = imported(py, PLANNER_MODELS, "PathSelector")?; + for selector in selectors { + for token in selector.call_method0("split")?.try_iter()? { + for part in token?.call_method1("split", (",",))?.try_iter()? { + let parsed: Bound<'_, PyAny> = parse.call1((part?,))?; + if parsed.is_instance(&path_selector)? + || parsed.getattr("upstream")?.is_truthy()? + || parsed.getattr("downstream")?.is_truthy()? + { + return Err(user_error( + py, + (GRAPH_OPERATOR_CODE, GRAPH_OPERATOR_MESSAGE), + )); + } + } + } + } + Ok(()) +} + +fn python_nodes<'py>( + graph: &Bound<'py, PyAny>, + (select, exclude): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + call_with( + &imported( + graph.py(), + PYTHON_SELECTORS_MODULE, + "resolve_python_nodes_from_selectors", + )?, + &[ + ("select", select), + ("exclude", exclude), + ("graph", graph.clone()), + ], + ) +} + +fn node_kind<'py>( + graph: &Bound<'py, PyAny>, + name: &Bound<'py, PyAny>, +) -> PyResult> { + graph + .getattr("nodes_by_name")? + .get_item(name)? + .getattr("kind") +} + +/// The selected check names: selected checks, or every check minus excluded nodes. +fn selected_check_names<'py>( + request: &Bound<'py, PyAny>, + graph: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = graph.py(); + let select: Bound<'py, PyAny> = request.getattr("select")?; + let exclude: Bound<'py, PyAny> = request.getattr("exclude")?; + let selectors: Vec> = select + .try_iter()? + .chain(exclude.try_iter()?) + .collect::>()?; + refuse_graph_selectors(py, &selectors)?; + let check_kind: Bound<'py, PyAny> = + imported(py, PYTHON_NODE_TYPES, "PythonNodeKind")?.getattr("CHECK")?; + if select.is_truthy()? { + let selected: Bound<'py, PyAny> = python_nodes(graph, (select, exclude))?; + let mut non_checks: Vec = Vec::new(); + for name in selected.try_iter()? { + let name: Bound<'py, PyAny> = name?; + if !node_kind(graph, &name)?.eq(&check_kind)? { + non_checks.push(name.extract()?); + } + } + validate_check_selection(&non_checks).map_err(|error| selection_error(py, &error))?; + return selected.downcast_into().map_err(Into::into); + } + let excluded: Bound<'py, PyAny> = if exclude.is_truthy()? { + python_nodes(graph, (exclude, empty_tuple(py)))? + } else { + PyFrozenSet::empty(py)?.into_any() + }; + let mut names: Vec> = Vec::new(); + for node in graph.getattr("nodes")?.try_iter()? { + let node: Bound<'py, PyAny> = node?; + let name: Bound<'py, PyAny> = node.getattr("name")?; + if node.getattr("kind")?.eq(&check_kind)? && !excluded.contains(&name)? { + names.push(name); + } + } + PyFrozenSet::new(py, names) +} + +/// Compile the SQL relations and loader sources the selected checks depend on. +fn compile_pipeline<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, + (graph, check_names): (&Bound<'py, PyAny>, &Bound<'py, PyFrozenSet>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; + let required_refs: Bound<'py, PyAny> = call_with( + &graph.getattr("selected_sql_refs")?, + &[("selected_names", check_names.clone().into_any())], + )?; + let closure: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "transitive_closure_many")?, + &[ + ("starts", check_names.clone().into_any()), + ("edges", graph.getattr("upstream_deps")?), + ( + "include_starts", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?; + let loader_kind: Bound<'py, PyAny> = + imported(py, PYTHON_NODE_TYPES, "PythonNodeKind")?.getattr("LOADER")?; + let mut loaders: Vec> = Vec::new(); + for name in closure.try_iter()? { + let name: Bound<'py, PyAny> = name?; + if !check_names.contains(&name)? && node_kind(graph, &name)?.eq(&loader_kind)? { + loaders.push(name); + } + } + let loaders: Bound<'py, PyFrozenSet> = PyFrozenSet::new(py, loaders)?; + let key_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS, "CompiledObjectKey")?; + let source_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + let ref_key: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "sql_ref_key")?; + let mut keys: Vec> = Vec::new(); + for reference in required_refs.try_iter()? { + keys.push(ref_key.call1((reference?,))?); + } + for source_file in discovered.getattr("source_files")?.try_iter()? { + for source in source_file?.getattr("source_entries")?.try_iter()? { + let source: Bound<'py, PyAny> = source?; + if loaders.contains(source.getattr("loader")?)? { + keys.push(call_with( + &key_class, + &[ + ("resource_type", source_type.clone()), + ("name", source.getattr("name")?), + ], + )?); + } + } + } + let keys: Bound<'py, PyAny> = PyFrozenSet::new(py, keys)?.into_any(); + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.getattr("effective_project_dir")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let options: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &[ + ("selected_target", request.getattr("selected_target")?), + ("no_sql_validation", request.getattr("no_sql_validation")?), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("cli_vars", request.getattr("cli_vars")?), + ("external_sql_reference_resolver", resolver), + ], + )?; + let context: Bound<'py, PyAny> = call_with( + &imported(py, STATIC_COMMAND_MODULE, "compile_static_command_context")?, + &[ + ("discovered_inputs", discovered), + ("adapter", invocation.getattr("adapter")?), + ("options", options), + ("selected_keys", keys.clone()), + ("relation_keys", keys), + ( + "on_progress", + invocation + .getattr("planning_progress")? + .getattr("on_progress")?, + ), + ], + )?; + let plan_output: Bound<'py, PyAny> = call_with( + &imported(py, RELATION_PLAN_MODULE, "build_relation_command_plan")?, + &[ + ("project", context.getattr("project")?), + ("scope", context.getattr("scope")?), + ("relations", context.getattr("relations")?), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "build_static_pipeline_result")?, + &[("context", context), ("plan_output", plan_output)], + ) +} + +fn target_default<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + (field, fallback): (&str, &str), +) -> PyResult> { + let value: Bound<'py, PyAny> = project.getattr(field)?; + if value.is_none() { + adapter.call_method0(fallback) + } else { + Ok(value) + } +} + +/// The discovered Python graph, after refusing unit-test selectors, and the selected checks. +fn python_selection<'py>( + request: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyFrozenSet>)> { + let py: Python<'py> = request.py(); + let python_graph: Bound<'py, PyAny> = call_with( + &imported( + py, + PYTHON_GRAPH_MODULE, + "build_discovered_python_node_graph", + )?, + &[("discovered_inputs", discovered.clone())], + )?; + call_with( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let check_names: Bound<'py, PyFrozenSet> = selected_check_names(request, &python_graph)?; + Ok((python_graph, check_names)) +} + +fn prepare<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; + let adapter: Bound<'py, PyAny> = invocation.getattr("adapter")?; + let (planning_graph, planning_names) = python_selection(request, &discovered)?; + let pipeline: Bound<'py, PyAny> = + compile_pipeline(request, invocation, (&planning_graph, &planning_names))?; + let (python_graph, check_names) = python_selection(request, &discovered)?; + let mut check_functions: Vec> = Vec::new(); + for check in discovered.getattr("check_functions")?.try_iter()? { + let check: Bound<'py, PyAny> = check?; + if check_names.contains(check.getattr("name")?)? { + check_functions.push(check); + } + } + let selection: Bound<'py, PyAny> = call_with( + &imported(py, PYTHON_NODE_MODELS, "PythonSqlRunSelection")?, + &[ + ("sql_keys", PyFrozenSet::empty(py)?.into_any()), + ("python_node_names", PyFrozenSet::empty(py)?.into_any()), + ], + )?; + let lifecycle_plan: Bound<'py, PyAny> = call_with( + &imported( + py, + PYTHON_LIFECYCLE_MODULE, + "build_python_sql_run_lifecycle", + )?, + &[ + ("selection", selection), + ("python_graph", python_graph.clone()), + ], + )?; + let relation_refs: Bound<'py, PyAny> = call_with( + &python_graph.getattr("selected_sql_refs")?, + &[("selected_names", check_names.clone().into_any())], + )?; + let project: Bound<'py, PyAny> = pipeline.getattr("project")?; + let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let relation_targets: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_python_relation_targets")?, + &[ + ("adapter", adapter.clone()), + ("project", project.clone()), + ("plan_output", plan_output.clone()), + ("required_refs", relation_refs.clone()), + ], + )?; + Ok(Prepared { + check_functions: PyTuple::new(py, check_functions)?.into_any(), + lifecycle_plan, + relation_targets, + relation_refs, + defaults: ( + target_default( + &project, + &adapter, + ("effective_target_database", "default_database"), + )?, + target_default( + &project, + &adapter, + ("effective_target_schema", "default_schema"), + )?, + ), + project_relations: call_with( + &imported(py, NATIVE_MODULE, "build_project_relation_targets")?, + &[("adapter", adapter), ("plan_output", plan_output)], + )?, + python_graph, + pipeline, + }) +} + +/// The executor runtime of check dependencies and checks; ingress runs without project relations. +fn node_runtime<'py>( + invocation: &Bound<'py, PyAny>, + prepared: &Prepared<'py>, + (connection, providers, with_project_relations): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, bool), +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let project: Bound<'py, PyAny> = prepared.pipeline.getattr("project")?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("adapter", invocation.getattr("adapter")?), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("connection", connection.clone()), + ("run_id", project.getattr("run_id")?), + ("target", project.getattr("effective_target_name")?), + ("vars", project.getattr("effective_vars")?), + ("is_reload", PyBool::new(py, false).to_owned().into_any()), + ("default_database", prepared.defaults.0.clone()), + ("default_schema", prepared.defaults.1.clone()), + ("relation_targets", prepared.relation_targets.clone()), + ]; + if with_project_relations { + fields.push(("project_relations", prepared.project_relations.clone())); + } + fields.push(("providers", providers.clone())); + call_with( + &imported(py, EXECUTOR_MODELS, "PythonNodeRuntime")?, + &fields, + ) +} + +fn record_run_state<'py>( + invocation: &Bound<'py, PyAny>, + (run_state, python_results, load_results): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Option>, + ), + source_map: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = invocation.py(); + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ( + "discovered_inputs", + invocation.getattr("discovered_inputs")?, + ), + ("run_state", run_state), + ("python_results", python_results), + ]; + if let Some(load_results) = load_results { + fields.push(("load_results", load_results)); + } + fields.push(("source_map", source_map.clone())); + call_with( + &imported(py, CHECK_CORE_MODULE, "record_python_run_state_results")?, + &fields, + )?; + Ok(()) +} + +/// Run ingress loaders and read-side dependencies, then the checks themselves. +fn run_checks<'py>( + invocation: &Bound<'py, PyAny>, + prepared: &Prepared<'py>, + (connection, providers, event_writer): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let plan_output: Bound<'py, PyAny> = prepared.pipeline.getattr("plan_output")?; + let source_map: Bound<'py, PyAny> = plan_output.getattr("source_map")?; + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTOR_MODELS, "IngressCallbacks")?, + &[("use_color", invocation.getattr("use_color")?)], + )?; + let ingress: Bound<'py, PyAny> = call_with( + &imported(py, INGRESS_MODULE, "execute_ingress_python_loader_nodes")?, + &[ + ("python_graph", prepared.python_graph.clone()), + ( + "selected_python_names", + prepared + .lifecycle_plan + .getattr("ingress_python_node_names")?, + ), + ( + "loader_functions", + invocation + .getattr("discovered_inputs")? + .getattr("loader_functions")?, + ), + ("source_map", source_map.clone()), + ( + "runtime", + node_runtime(invocation, prepared, (connection, providers, false))?, + ), + ("callbacks", callbacks), + ], + )?; + let run_state: Bound<'py, PyAny> = ingress.getattr("run_state")?; + let load_results: Bound<'py, PyAny> = ingress.getattr("load_results")?; + let ingress_python: Bound<'py, PyAny> = ingress.getattr("python_results")?; + record_run_state( + invocation, + ( + run_state.clone(), + ingress_python.clone(), + Some(load_results.clone()), + ), + &source_map, + )?; + let read_side: Bound<'py, PyAny> = call_with( + &imported(py, CHECK_CORE_MODULE, "run_check_read_side_dependencies")?, + &[ + ("adapter", invocation.getattr("adapter")?), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("connection", connection.clone()), + ("pipeline_result", prepared.pipeline.clone()), + ("python_graph", prepared.python_graph.clone()), + ("lifecycle_plan", prepared.lifecycle_plan.clone()), + ("relation_targets", prepared.relation_targets.clone()), + ("validation_refs", prepared.relation_refs.clone()), + ("project_relations", prepared.project_relations.clone()), + ("providers", providers.clone()), + ], + )?; + record_run_state( + invocation, + (run_state.clone(), read_side.clone(), None), + &source_map, + )?; + let upstream: Vec> = ingress_python + .try_iter()? + .chain(read_side.try_iter()?) + .collect::>()?; + let by_loader: Bound<'py, PyAny> = call_with( + &imported(py, CHECK_CORE_MODULE, "load_results_by_loader_name")?, + &[ + ("source_map", source_map), + ("load_results", load_results.clone()), + ], + )?; + let on_complete: Bound<'py, PyAny> = Bound::new( + py, + CheckCompletionRelay { + event_writer: event_writer.clone().unbind(), + plan_output: plan_output.clone().unbind(), + }, + )? + .into_any(); + let check_callbacks: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTOR_MODELS, "PythonCheckCallbacks")?, + &[("on_check_complete", on_complete)], + )?; + call_with( + &imported(py, CHECKS_MODULE, "execute_python_checks")?, + &[ + ("check_functions", prepared.check_functions.clone()), + ("python_graph", prepared.python_graph.clone()), + ( + "upstream_python_results", + PyTuple::new(py, upstream)?.into_any(), + ), + ("upstream_load_results", load_results), + ("upstream_load_results_by_loader_name", by_loader), + ( + "runtime", + node_runtime(invocation, prepared, (connection, providers, true))?, + ), + ("run_state", run_state), + ( + "require_upstream_results", + PyBool::new(py, false).to_owned().into_any(), + ), + ("callbacks", check_callbacks), + ], + ) +} + +/// Record each finished check in the event output. +#[pyclass(module = "sqlbuild._native", frozen)] +struct CheckCompletionRelay { + event_writer: Py, + plan_output: Py, +} + +#[pymethods] +impl CheckCompletionRelay { + fn __call__(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("result", result)?; + options.set_item("plan", self.plan_output.bind(py))?; + options.set_item("command", "check")?; + self.event_writer + .bind(py) + .call_method("write_build_result", (), Some(&options))?; + Ok(()) + } +} + +/// Connect, configure progress ordinals, run the checks, and close everything. +fn execute<'py>( + invocation: &Bound<'py, PyAny>, + prepared: &Prepared<'py>, + providers: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = invocation.py(); + let adapter: Bound<'py, PyAny> = invocation.getattr("adapter")?; + let connection: Bound<'py, PyAny> = + adapter.call_method1("connect", (invocation.getattr("connection_config")?,))?; + let event_writer: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; + let executed: PyResult>> = (|| { + let projector: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + if !projector.is_none() { + let ordinals: Bound<'py, PyDict> = PyDict::new(py); + let mut total: usize = 0; + for check in prepared.check_functions.try_iter()? { + total += 1; + ordinals.set_item(check?.getattr("name")?, total)?; + } + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("ordinals", ordinals)?; + options.set_item("total", total)?; + projector.call_method("configure_resources", (), Some(&options))?; + } + run_checks( + invocation, + prepared, + (&connection, providers, &event_writer), + )? + .try_iter()? + .collect() + })(); + let writer_closed: PyResult> = event_writer.call_method0("close"); + let connection_closed: PyResult> = + adapter.call_method1("close", (&connection,)); + let results: Vec> = executed?; + writer_closed?; + connection_closed?; + Ok(results) +} + +/// Write rows, the summary, the runtime artifact and JSON output; returns the exit code. +fn complete<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, + (prepared, results): (&Prepared<'py>, &[Bound<'py, PyAny>]), +) -> PyResult { + let py: Python<'py> = request.py(); + let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; + let style: CliStyle = CliStyle { + use_color: invocation.getattr("use_color")?.extract()?, + }; + write_results( + &stream, + results, + (&prepared.check_functions, &prepared.python_graph), + style, + )?; + let texts: Vec = results + .iter() + .map(check_result_text) + .collect::>()?; + stream.call_method1("write", (render_check_summary(&texts, style),))?; + stream.call_method0("flush")?; + let tuple: Bound<'py, PyAny> = PyTuple::new(py, results)?.into_any(); + call_with( + &imported( + py, + RUNTIME_TARGET_MODULE, + "write_python_check_runtime_target", + )?, + &[ + ( + "target_dir", + invocation + .getattr("effective_project_dir")? + .call_method1("__truediv__", ("target",))?, + ), + ("results", tuple), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ( + "payload", + PyString::new(py, &results_json(results)?).into_any(), + ), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(check_exit_code(&texts)) +} + +/// Run one `sqb check` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let invocation: Bound<'_, PyAny> = call_with( + &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, + &[ + ("request", request.clone()), + ( + "invocation_type", + imported(py, COMMAND_MODELS, "CheckInvocation")?, + ), + ], + )?; + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + stream.call_method1("write", ("\n",))?; + let header: Bound<'_, PyDict> = PyDict::new(py); + header.set_item("stream", &stream)?; + header.set_item("command", EXECUTION_COMMAND)?; + header.set_item("target", py.None())?; + header.set_item("concurrency", PyInt::new(py, 1))?; + header.set_item("use_color", invocation.getattr("use_color")?)?; + imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&header))?; + let prepared: Prepared<'_> = prepare(request, &invocation)?; + let session: Bound<'_, PyAny> = call_with( + &imported(py, PROVIDER_SESSION_MODULE, "build_provider_session")?, + &[( + "discovered_providers", + invocation + .getattr("discovered_inputs")? + .getattr("providers")?, + )], + )?; + let outcome: PyResult = (|| { + let results: Vec> = + execute(&invocation, &prepared, &session.getattr("providers")?)?; + complete(request, &invocation, (&prepared, &results)) + })(); + let closed: PyResult> = session.call_method0("close"); + let code: i32 = outcome?; + closed?; + Ok(code) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_check_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs new file mode 100644 index 0000000000..cf21b50935 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs @@ -0,0 +1,230 @@ +//! Python check results as grouped rows and as the `sqb check --json` payload. + +use std::collections::HashMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::check_command::main::check_group_header::check_group_header; +use sqlbuild_core::check_command::main::check_group_label::check_group_label; +use sqlbuild_core::check_command::main::check_json::check_json; +use sqlbuild_core::check_command::main::render_check_row::render_check_row; +use sqlbuild_core::check_command::models::{CheckGrouping, CheckResultText, CheckRow, CheckStatus}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::json::models::JsonValue; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::result_output::integration_values::json_value; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const WARN_SEVERITY: &str = "warn"; + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +/// The metadata as JSON, through `json.dumps` when it holds values only that serializer accepts. +fn metadata_json(metadata: &Bound<'_, PyAny>) -> PyResult { + if let Some(value) = json_value(metadata)? { + return Ok(value); + } + let py: Python<'_> = metadata.py(); + let json: Bound<'_, PyAny> = py.import("json")?.into_any(); + let round_trip: Bound<'_, PyAny> = + json.call_method1("loads", (json.call_method1("dumps", (metadata,))?,))?; + json_value(&round_trip)? + .ok_or_else(|| PyValueError::new_err("check metadata is not JSON serializable")) +} + +/// One executor `PythonCheckExecutionResult` as the command renders it. +pub(crate) fn check_result_text(result: &Bound<'_, PyAny>) -> PyResult { + let severity: String = result.getattr("severity")?.getattr("value")?.extract()?; + let status: CheckStatus = if result.getattr("passed")?.is_truthy()? { + CheckStatus::Pass + } else if severity == WARN_SEVERITY { + CheckStatus::Warn + } else { + CheckStatus::Fail + }; + Ok(CheckResultText { + name: result.getattr("node_name")?.extract()?, + status, + severity, + message: optional_text(result.getattr("message")?)?, + error_message: optional_text(result.getattr("error_message")?)?, + metadata: metadata_json(&result.getattr("metadata")?)?, + warning_messages: result.getattr("warning_messages")?.extract()?, + }) +} + +fn names(values: Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +/// The output group of `name`, from the Python graph and the check's declared group. +fn grouping( + name: &str, + checks: &HashMap>, + python_graph: &Bound<'_, PyAny>, +) -> PyResult { + let group: Option = match checks.get(name) { + Some(check) => optional_text(check.getattr("group")?)?, + None => None, + }; + if python_graph.is_none() { + return Ok(CheckGrouping { + dependencies: None, + group, + }); + } + let py: Python<'_> = python_graph.py(); + let empty: Bound<'_, PyAny> = pyo3::types::PyTuple::empty(py).into_any(); + let mut dependencies: Vec = names( + python_graph + .getattr("upstream_deps")? + .call_method1("get", (name, empty))?, + )?; + let node: Bound<'_, PyAny> = python_graph + .getattr("nodes_by_name")? + .call_method1("get", (name,))?; + if !node.is_none() { + for reference in node.getattr("sql_deps")?.try_iter()? { + dependencies.push(reference?.getattr("name")?.extract()?); + } + } + Ok(CheckGrouping { + dependencies: Some(dependencies), + group, + }) +} + +/// Write grouped check rows, claiming each row's projected duration once its row is written. +pub(crate) fn write_results( + stream: &Bound<'_, PyAny>, + results: &[Bound<'_, PyAny>], + (check_functions, python_graph): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + style: CliStyle, +) -> PyResult<()> { + let py: Python<'_> = stream.py(); + let mut checks: HashMap> = HashMap::new(); + for check in check_functions.try_iter()? { + let check: Bound<'_, PyAny> = check?; + checks.insert(check.getattr("name")?.extract()?, check); + } + let projector: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + let mut groups: Vec<(String, Vec)> = Vec::new(); + for result in results { + let text: CheckResultText = check_result_text(result)?; + let label: String = check_group_label(&grouping(&text.name, &checks, python_graph)?); + match groups.iter_mut().find(|(group, _)| *group == label) { + Some((_, members)) => members.push(text), + None => groups.push((label, vec![text])), + } + } + stream.call_method1("write", ("\n",))?; + for (group, members) in groups { + stream.call_method1("write", (check_group_header(&group),))?; + for result in members { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", &result.name)?; + options.set_item("resource_id", format!("check:{}", result.name))?; + let duration_ms: Option = if projector.is_none() { + None + } else { + projector + .call_method("resource_terminal_duration", (), Some(&options))? + .extract()? + }; + let row: CheckRow = CheckRow { + result, + duration_ms, + }; + stream.call_method1("write", (render_check_row(&row, style),))?; + if !projector.is_none() { + projector.call_method("consume_resource_terminal", (), Some(&options))?; + } + } + stream.call_method1("write", ("\n",))?; + } + Ok(()) +} + +/// The JSON payload of the results the terminal event index saw finish, or of all of them. +pub(crate) fn results_json(results: &[Bound<'_, PyAny>]) -> PyResult { + let Some(first) = results.first() else { + return check_json(&[]).map_err(|error| PyValueError::new_err(format!("{error:?}"))); + }; + let py: Python<'_> = first.py(); + let index: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "current_terminal_event_index")?.call0()?; + let mut texts: Vec = Vec::new(); + for result in results { + let name: Bound<'_, PyAny> = result.getattr("node_name")?; + if !index.is_none() { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", &name)?; + options.set_item( + "resource_id", + PyString::new(py, &format!("check:{}", name.str()?)), + )?; + if index + .call_method("resource_terminal", (), Some(&options))? + .is_none() + { + continue; + } + } + texts.push(check_result_text(result)?); + } + check_json(&texts).map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + +/// Write human-readable check result rows. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_check_results(py: Python<'_>, options: Option>) -> PyResult<()> { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "write_check_results", + options, + ( + &["stream", "results", "use_color"], + &["check_functions", "python_graph"], + ), + )?; + let results: Vec> = arguments + .value("results")? + .try_iter()? + .collect::>()?; + let check_functions: Bound<'_, PyAny> = match arguments.value("check_functions")? { + functions if functions.is_none() => pyo3::types::PyTuple::empty(py).into_any(), + functions => functions, + }; + write_results( + &arguments.value("stream")?, + &results, + (&check_functions, &arguments.value("python_graph")?), + CliStyle { + use_color: arguments.flag("use_color")?, + }, + ) +} + +/// Format check execution JSON. +#[pyfunction] +#[pyo3(signature = (*, results))] +fn format_check_json(results: &Bound<'_, PyAny>) -> PyResult { + let results: Vec> = results.try_iter()?.collect::>()?; + results_json(&results) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_check_results, module)?)?; + module.add_function(wrap_pyfunction!(format_check_json, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs new file mode 100644 index 0000000000..3bde938874 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod check_command; +pub(crate) mod check_results; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs new file mode 100644 index 0000000000..5baf04cec2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs @@ -0,0 +1,10 @@ +//! Register the native `sqb check` command and its result rendering. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::check::{check_command, check_results}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + check_command::register(module)?; + check_results::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 9508da8e0a..c2c3ce805a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -4,8 +4,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; use crate::bindings::_helpers::{ - clone, command_runtime, cost_report, dbt_interop, debug_command, freshness, janitor, load, - presentation_text, result_output, workspace_commands, + check, clone, command_runtime, cost_report, dbt_interop, debug_command, freshness, janitor, + load, presentation_text, result_output, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -25,5 +25,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { command_runtime::registration::register(module)?; presentation_text::registration::register(module)?; workspace_commands::registration::register(module)?; + check::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index abcc940d0a..e26581ccea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod auditing; pub(crate) mod authored_values; pub(crate) mod boundary; pub(crate) mod cache; +pub(crate) mod check; pub(crate) mod cli; pub(crate) mod clone; pub(crate) mod command_runtime; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 590b06053b..3d0c05ab19 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -19,6 +19,7 @@ from sqlbuild.adapter.contract.types import ( from sqlbuild.adapter.relations.models import InspectionQueryRecord from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.models import ( + CheckCommandRequest, CloneCommandRequest, FreshnessCommandRequest, JanitorCommandRequest, @@ -110,6 +111,7 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredAdapterFile, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredCheckFunction, DiscoveredConstantFile, DiscoveredEnumFile, DiscoveredHookFunction, @@ -2829,6 +2831,16 @@ def run_janitor_command(*, request: JanitorCommandRequest) -> int: ... def run_freshness_command(*, request: FreshnessCommandRequest) -> int: ... def run_clone_command(*, request: CloneCommandRequest) -> int: ... def run_load_command(*, request: LoadCommandRequest) -> int: ... +def run_check_command(*, request: CheckCommandRequest) -> int: ... +def write_check_results( + *, + stream: TextIO, + results: tuple[PythonCheckExecutionResult, ...], + use_color: bool, + check_functions: tuple[DiscoveredCheckFunction, ...] = (), + python_graph: PythonNodeGraph | None = None, +) -> None: ... +def format_check_json(*, results: tuple[PythonCheckExecutionResult, ...]) -> str: ... def select_load_entries( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py b/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py index acb0929b8d..2e302edb70 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py @@ -4,13 +4,12 @@ from typing import Any -from sqlbuild._native import build_project_relation_targets +from sqlbuild._native import build_project_relation_targets, write_check_results from sqlbuild.cli.commands._helpers.check.core import ( build_check_relation_targets, load_results_by_loader_name, record_python_run_state_results, relevant_check_functions, - write_check_results, ) from sqlbuild.cli.commands.models import ( BuildCommandRequest, diff --git a/src/sqlbuild/cli/commands/_helpers/check/core.py b/src/sqlbuild/cli/commands/_helpers/check/core.py index 6e84a0da06..3ff34e394d 100644 --- a/src/sqlbuild/cli/commands/_helpers/check/core.py +++ b/src/sqlbuild/cli/commands/_helpers/check/core.py @@ -2,17 +2,11 @@ from __future__ import annotations -import json from collections.abc import Callable -from typing import Any, TextIO +from typing import Any from sqlbuild._native import ( - NativeProgressProjector, - TerminalEventIndex, build_python_relation_targets, - current_native_progress_projector, - current_terminal_event_index, - transitive_closure_many, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup @@ -25,11 +19,8 @@ from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector from sqlbuild.compiler.python_nodes.main.selectors import resolve_python_nodes_from_selectors from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonNode, PythonNodeGraph, PythonSqlRunLifecyclePlan, ) @@ -43,7 +34,6 @@ PythonNodeRunState, PythonNodeRuntime, ) -from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.provider.main.runtime import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind @@ -52,74 +42,6 @@ _IGNORED_EXCLUDE_SELECTOR_ERROR_CODES: frozenset[str] = frozenset({"S007", "S008", "S009"}) -def resolve_selected_check_names( - *, graph: PythonNodeGraph, select: tuple[str, ...], exclude: tuple[str, ...] -) -> frozenset[str]: - """Resolve sqb check selectors into check names only.""" - - _validate_check_selectors(select=(*select, *exclude)) - if select: - selected_names: frozenset[str] = resolve_python_nodes_from_selectors( - select=select, exclude=exclude, graph=graph - ) - return _check_names_from_selection(graph=graph, selected_names=selected_names) - check_names: frozenset[str] = frozenset( - node.name for node in graph.nodes if node.kind == PythonNodeKind.CHECK - ) - excluded: frozenset[str] = ( - resolve_python_nodes_from_selectors(select=exclude, exclude=(), graph=graph) - if exclude - else frozenset() - ) - return frozenset(check_names - excluded) - - -def check_dependency_closure( - *, graph: PythonNodeGraph, check_names: frozenset[str] -) -> frozenset[str]: - """Return transitive executable Python dependencies of selected checks.""" - - dependencies: frozenset[str] = transitive_closure_many( - starts=check_names, edges=graph.upstream_deps, include_starts=False - ) - return dependencies - check_names - - -def _validate_check_selectors(*, select: tuple[str, ...]) -> None: - raw_selector: str - for raw_selector in select: - token: str - for token in raw_selector.split(): - part: str - for part in token.split(","): - parsed: ParsedSelector | PathSelector = parse_project_selector(part) - if isinstance(parsed, PathSelector) or parsed.upstream or parsed.downstream: - raise CliUserError( - "sqb check selectors do not support graph operators; " - "select checks directly without '+', '~', or path-between expansion", - code="C681", - ) - - -def _check_names_from_selection( - *, graph: PythonNodeGraph, selected_names: frozenset[str] -) -> frozenset[str]: - non_check_names: tuple[str, ...] = tuple( - sorted( - name - for name in selected_names - if graph.nodes_by_name[name].kind != PythonNodeKind.CHECK - ) - ) - if non_check_names: - raise CliUserError( - "sqb check selectors must resolve only to Python checks; non-check selections: " - + ", ".join(non_check_names), - code="C682", - ) - return selected_names - - def record_python_run_state_results( *, discovered_inputs: DiscoveredProjectInputs, @@ -265,99 +187,6 @@ def load_results_by_loader_name( } -def write_check_results( - *, - stream: TextIO, - results: tuple[PythonCheckExecutionResult, ...], - use_color: bool, - check_functions: tuple[DiscoveredCheckFunction, ...] = (), - python_graph: PythonNodeGraph | None = None, -) -> None: - """Write human-readable check result rows.""" - - style: CliStyle = CliStyle(use_color=use_color) - projector: NativeProgressProjector | None = current_native_progress_projector() - result_by_name: dict[str, PythonCheckExecutionResult] = { - result.node_name: result for result in results - } - grouped_names: dict[str, list[str]] = _group_check_names( - results=results, - check_functions=check_functions, - python_graph=python_graph, - ) - stream.write("\n") - group_label: str - result_names: list[str] - for group_label, result_names in grouped_names.items(): - stream.write(f"Python checks: {group_label}\n\n") - for result_name in result_names: - result: PythonCheckExecutionResult = result_by_name[result_name] - duration_ms: float | None = ( - projector.resource_terminal_duration( - resource_name=result.node_name, - resource_id=f"check:{result.node_name}", - ) - if projector is not None - else None - ) - _write_check_result_row( - stream=stream, - result=result, - style=style, - duration_ms=duration_ms, - ) - if projector is not None: - _ = projector.consume_resource_terminal( - resource_name=result.node_name, - resource_id=f"check:{result.node_name}", - ) - stream.write("\n") - - -def format_check_json(*, results: tuple[PythonCheckExecutionResult, ...]) -> str: - """Format check execution JSON.""" - - projector: TerminalEventIndex | None = current_terminal_event_index() - if projector is not None: - results = tuple( - result - for result in results - if projector.resource_terminal( - resource_name=result.node_name, - resource_id=f"check:{result.node_name}", - ) - is not None - ) - payload: dict[str, object] = { - "version": 1, - "command": "check", - "status": "failed" if check_results_failed(results) else "success", - "summary": { - "pass_count": sum(1 for result in results if result.passed), - "warn_count": sum(1 for result in results if result.warned), - "fail_count": sum(1 for result in results if result.failed), - "total_count": len(results), - }, - "checks": [ - { - "kind": "python_check", - "name": result.node_name, - "display_name": result.node_name, - "check_id": f"python_check:{result.node_name}", - "status": "pass" if result.passed else "warn" if result.warned else "fail", - "passed": result.passed, - "severity": result.severity.value, - "message": result.message, - "error_message": result.error_message, - "metadata": result.metadata, - "warnings": list(result.warning_messages) or None, - } - for result in results - ], - } - return json.dumps(payload, indent=2) + "\n" - - def check_results_failed(results: tuple[PythonCheckExecutionResult, ...]) -> bool: """Return whether any Python check produced an error-severity failure.""" @@ -528,67 +357,3 @@ def _check_sql_ref_location( def _sql_ref_sort_key(ref: SqlResourceRef) -> tuple[str, str]: return (ref.kind.value, ref.name) - - -def _group_check_names( - *, - results: tuple[PythonCheckExecutionResult, ...], - check_functions: tuple[DiscoveredCheckFunction, ...], - python_graph: PythonNodeGraph | None, -) -> dict[str, list[str]]: - check_by_name: dict[str, DiscoveredCheckFunction] = { - check.name: check for check in check_functions - } - groups: dict[str, list[str]] = {} - result: PythonCheckExecutionResult - for result in results: - group_label: str = _check_group_label( - check=check_by_name.get(result.node_name), - result_name=result.node_name, - python_graph=python_graph, - ) - groups.setdefault(group_label, []).append(result.node_name) - return groups - - -def _check_group_label( - *, - check: DiscoveredCheckFunction | None, - result_name: str, - python_graph: PythonNodeGraph | None, -) -> str: - if python_graph is not None: - node: DiscoveredPythonNode | None = python_graph.nodes_by_name.get(result_name) - upstream_names: tuple[str, ...] = ( - *python_graph.upstream_deps.get(result_name, ()), - *(() if node is None else (sql_ref.name for sql_ref in node.sql_deps)), - ) - if len(upstream_names) == 1: - return upstream_names[0] - if len(upstream_names) > 1: - return ( - check.group if check is not None and check.group is not None else "multi-dependency" - ) - if check is not None and check.group is not None: - return check.group - return "ungrouped" - - -def _write_check_result_row( - *, - stream: TextIO, - result: PythonCheckExecutionResult, - style: CliStyle, - duration_ms: float | None, -) -> None: - status: str = "PASS" if result.passed else "WARN" if result.warned else "FAIL" - stream.write(f" {'check':<10}{result.node_name:<50} {style.status(status=status)}") - if duration_ms is not None: - stream.write(f" {duration_ms / 1000.0:.2f}s") - detail: str | None = result.error_message or result.message - if detail: - stream.write(f" {detail}") - stream.write("\n") - warning_message: str - for warning_message in result.warning_messages: - stream.write(f" {style.warning(warning_message)}\n") diff --git a/src/sqlbuild/cli/commands/_helpers/check/execution.py b/src/sqlbuild/cli/commands/_helpers/check/execution.py deleted file mode 100644 index 973e138dbd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/check/execution.py +++ /dev/null @@ -1,272 +0,0 @@ -"""Check command execution phases.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild._native import ( - ExecutionEventWriter, - NativeProgressProjector, - build_project_relation_targets, - current_native_progress_projector, -) -from sqlbuild.cli.commands._helpers.check.core import ( - build_check_relation_targets, - load_results_by_loader_name, - record_python_run_state_results, - resolve_selected_check_names, - run_check_read_side_dependencies, -) -from sqlbuild.cli.commands.models import ( - CheckCommandRequest, - CheckExecutionPreparation, - CheckInvocation, -) -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.main.run_lifecycle import build_python_sql_run_lifecycle -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) -from sqlbuild.executor.python_nodes.main.checks import execute_python_checks -from sqlbuild.executor.python_nodes.main.ingress import execute_ingress_python_loader_nodes -from sqlbuild.executor.python_nodes.models import ( - IngressCallbacks, - PythonCheckCallbacks, - PythonCheckExecutionResult, - PythonIngressLoaderExecutorResult, - PythonNodeExecutionResult, - PythonNodeRuntime, -) -from sqlbuild.provider.classes.container import ProviderContainer -from sqlbuild.python_nodes.models import SqlResourceRef - - -def prepare_check_execution( - *, - request: CheckCommandRequest, - invocation: CheckInvocation, - pipeline_result: CompilePipelineResult, -) -> CheckExecutionPreparation: - """Prepare Python graph selection and relation context for check execution.""" - - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=invocation.discovered_inputs - ) - reject_unit_test_selectors( - select=request.select, - exclude=request.exclude, - discovered_inputs=invocation.discovered_inputs, - ) - check_names: frozenset[str] = resolve_selected_check_names( - graph=python_graph, - select=request.select, - exclude=request.exclude, - ) - check_functions: tuple[DiscoveredCheckFunction, ...] = tuple( - check for check in invocation.discovered_inputs.check_functions if check.name in check_names - ) - lifecycle_plan: PythonSqlRunLifecyclePlan = build_python_sql_run_lifecycle( - selection=PythonSqlRunSelection( - sql_keys=frozenset(), - python_node_names=frozenset(), - ), - python_graph=python_graph, - ) - relation_targets: dict[SqlResourceRef, str] = build_check_relation_targets( - adapter=invocation.adapter, - pipeline_result=pipeline_result, - python_graph=python_graph, - selected_python_names=check_names, - ) - relation_refs: frozenset[SqlResourceRef] = python_graph.selected_sql_refs( - selected_names=check_names - ) - return CheckExecutionPreparation( - python_graph=python_graph, - check_functions=check_functions, - lifecycle_plan=lifecycle_plan, - relation_targets=relation_targets, - relation_refs=relation_refs, - default_database=_default_database(invocation=invocation, pipeline_result=pipeline_result), - default_schema=_default_schema(invocation=invocation, pipeline_result=pipeline_result), - project_relations=build_project_relation_targets( - adapter=invocation.adapter, plan_output=pipeline_result.plan_output - ), - ) - - -def execute_check_plan( - *, - invocation: CheckInvocation, - pipeline_result: CompilePipelineResult, - preparation: CheckExecutionPreparation, - providers: ProviderContainer, -) -> tuple[PythonCheckExecutionResult, ...]: - """Execute selected Python checks with their Python dependencies.""" - - connection: Any = invocation.adapter.connect(invocation.connection_config) - event_writer: ExecutionEventWriter = ExecutionEventWriter() - projector: NativeProgressProjector | None = current_native_progress_projector() - if projector is not None: - projector.configure_resources( - ordinals={ - check.name: index - for index, check in enumerate(preparation.check_functions, start=1) - }, - total=len(preparation.check_functions), - ) - on_check_complete: Callable[[PythonCheckExecutionResult], None] = _check_completion_adapter( - event_writer=event_writer, - pipeline_result=pipeline_result, - ) - try: - ingress_result: PythonIngressLoaderExecutorResult = _execute_check_ingress( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - connection=connection, - providers=providers, - ) - read_side_results: tuple[PythonNodeExecutionResult, ...] = _execute_check_read_side( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - connection=connection, - ingress_result=ingress_result, - providers=providers, - ) - return execute_python_checks( - check_functions=preparation.check_functions, - python_graph=preparation.python_graph, - upstream_python_results=(*ingress_result.python_results, *read_side_results), - upstream_load_results=ingress_result.load_results, - upstream_load_results_by_loader_name=load_results_by_loader_name( - source_map=pipeline_result.plan_output.source_map, - load_results=ingress_result.load_results, - ), - runtime=PythonNodeRuntime( - adapter=invocation.adapter, - connection_config=invocation.connection_config, - connection=connection, - run_id=pipeline_result.project.run_id, - target=pipeline_result.project.effective_target_name, - vars=pipeline_result.project.effective_vars, - is_reload=False, - default_database=preparation.default_database, - default_schema=preparation.default_schema, - relation_targets=preparation.relation_targets, - project_relations=preparation.project_relations, - providers=providers, - ), - run_state=ingress_result.run_state, - require_upstream_results=False, - callbacks=PythonCheckCallbacks(on_check_complete=on_check_complete), - ) - finally: - event_writer.close() - invocation.adapter.close(connection) - - -def _check_completion_adapter( - *, - event_writer: ExecutionEventWriter, - pipeline_result: CompilePipelineResult, -) -> Callable[[PythonCheckExecutionResult], None]: - def _on_complete(result: PythonCheckExecutionResult) -> None: - event_writer.write_build_result( - result=result, - plan=pipeline_result.plan_output, - command="check", - ) - - return _on_complete - - -def _execute_check_ingress( - *, - invocation: CheckInvocation, - pipeline_result: CompilePipelineResult, - preparation: CheckExecutionPreparation, - connection: Any, - providers: ProviderContainer, -) -> PythonIngressLoaderExecutorResult: - ingress_result: PythonIngressLoaderExecutorResult = execute_ingress_python_loader_nodes( - python_graph=preparation.python_graph, - selected_python_names=preparation.lifecycle_plan.ingress_python_node_names, - loader_functions=invocation.discovered_inputs.loader_functions, - source_map=pipeline_result.plan_output.source_map, - runtime=PythonNodeRuntime( - adapter=invocation.adapter, - connection_config=invocation.connection_config, - connection=connection, - run_id=pipeline_result.project.run_id, - target=pipeline_result.project.effective_target_name, - vars=pipeline_result.project.effective_vars, - is_reload=False, - default_database=preparation.default_database, - default_schema=preparation.default_schema, - relation_targets=preparation.relation_targets, - providers=providers, - ), - callbacks=IngressCallbacks(use_color=invocation.use_color), - ) - record_python_run_state_results( - discovered_inputs=invocation.discovered_inputs, - run_state=ingress_result.run_state, - python_results=ingress_result.python_results, - load_results=ingress_result.load_results, - source_map=pipeline_result.plan_output.source_map, - ) - return ingress_result - - -def _execute_check_read_side( - *, - invocation: CheckInvocation, - pipeline_result: CompilePipelineResult, - preparation: CheckExecutionPreparation, - connection: Any, - ingress_result: PythonIngressLoaderExecutorResult, - providers: ProviderContainer, -) -> tuple[PythonNodeExecutionResult, ...]: - read_side_results: tuple[PythonNodeExecutionResult, ...] = run_check_read_side_dependencies( - adapter=invocation.adapter, - connection_config=invocation.connection_config, - connection=connection, - pipeline_result=pipeline_result, - python_graph=preparation.python_graph, - lifecycle_plan=preparation.lifecycle_plan, - relation_targets=preparation.relation_targets, - validation_refs=preparation.relation_refs, - project_relations=preparation.project_relations, - providers=providers, - ) - record_python_run_state_results( - discovered_inputs=invocation.discovered_inputs, - run_state=ingress_result.run_state, - python_results=read_side_results, - source_map=pipeline_result.plan_output.source_map, - ) - return read_side_results - - -def _default_database( - *, invocation: CheckInvocation, pipeline_result: CompilePipelineResult -) -> str | None: - if pipeline_result.project.effective_target_database is not None: - return pipeline_result.project.effective_target_database - return invocation.adapter.default_database() - - -def _default_schema( - *, invocation: CheckInvocation, pipeline_result: CompilePipelineResult -) -> str | None: - if pipeline_result.project.effective_target_schema is not None: - return pipeline_result.project.effective_target_schema - return invocation.adapter.default_schema() diff --git a/src/sqlbuild/cli/commands/_helpers/check/invocation.py b/src/sqlbuild/cli/commands/_helpers/check/invocation.py deleted file mode 100644 index e0d0b8556d..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/check/invocation.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Check command invocation resolution phase.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( - resolve_reported_command_invocation, -) -from sqlbuild.cli.commands.models import CheckCommandRequest, CheckInvocation - - -def resolve_check_invocation(*, request: CheckCommandRequest) -> CheckInvocation: - """Resolve discovery, adapter, connection, and reporters for check.""" - - return resolve_reported_command_invocation(request=request, invocation_type=CheckInvocation) diff --git a/src/sqlbuild/cli/commands/_helpers/check/outputs.py b/src/sqlbuild/cli/commands/_helpers/check/outputs.py deleted file mode 100644 index ffc4034ea7..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/check/outputs.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Check command output writing and exit-code phases.""" - -from __future__ import annotations - -from sqlbuild._native import format_summary_footer, write_execution_json_output -from sqlbuild.cli.commands._helpers.check.core import format_check_json, write_check_results -from sqlbuild.cli.commands.models import ( - CheckCommandRequest, - CheckExecutionPreparation, - CheckInvocation, -) -from sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target import ( - write_python_check_runtime_target, -) -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult - - -def write_check_completion_output( - *, - request: CheckCommandRequest, - invocation: CheckInvocation, - preparation: CheckExecutionPreparation, - results: tuple[PythonCheckExecutionResult, ...], -) -> None: - """Write check rows, summary, runtime artifact, and optional JSON output.""" - - write_check_results( - stream=invocation.progress_stream, - results=results, - use_color=invocation.use_color, - check_functions=preparation.check_functions, - python_graph=preparation.python_graph, - ) - invocation.progress_stream.write( - "\n" - + format_summary_footer( - counts=( - ("PASS", _pass_count(results)), - ("WARN", _warn_count(results=results)), - ("FAIL", _fail_count(results)), - ("TOTAL", len(results)), - ), - use_color=invocation.use_color, - ) - + "\n" - ) - invocation.progress_stream.flush() - write_python_check_runtime_target( - target_dir=invocation.effective_project_dir / "target", results=results - ) - write_execution_json_output( - payload=format_check_json(results=results), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def resolve_check_exit_code(results: tuple[PythonCheckExecutionResult, ...]) -> int: - """Resolve the check exit code from failed Python check results.""" - - return 0 if _fail_count(results) == 0 else 1 - - -def _pass_count(results: tuple[PythonCheckExecutionResult, ...]) -> int: - return sum(1 for result in results if result.passed) - - -def _warn_count(results: tuple[PythonCheckExecutionResult, ...]) -> int: - return sum(1 for result in results if result.warned) - - -def _fail_count(results: tuple[PythonCheckExecutionResult, ...]) -> int: - return sum(1 for result in results if result.failed) diff --git a/src/sqlbuild/cli/commands/_helpers/check/planning.py b/src/sqlbuild/cli/commands/_helpers/check/planning.py deleted file mode 100644 index 629d3d9987..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/check/planning.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Check command plan compilation phase.""" - -from __future__ import annotations - -from sqlbuild._native import build_static_pipeline_result, sql_ref_key -from sqlbuild.cli.commands._helpers.check.core import ( - check_dependency_closure, - resolve_selected_check_names, -) -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.models import CheckCommandRequest, CheckInvocation -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, - StaticCommandContext, -) -from sqlbuild.compiler.planner.main.commands.relation_plan import build_relation_command_plan -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.python_nodes.models import SqlResourceRef - - -def compile_check_plan( - *, request: CheckCommandRequest, invocation: CheckInvocation -) -> CompilePipelineResult: - """Compile the project for Python check execution.""" - - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=invocation.discovered_inputs - ) - reject_unit_test_selectors( - select=request.select, - exclude=request.exclude, - discovered_inputs=invocation.discovered_inputs, - ) - check_names: frozenset[str] = resolve_selected_check_names( - graph=python_graph, - select=request.select, - exclude=request.exclude, - ) - required_refs: frozenset[SqlResourceRef] = python_graph.selected_sql_refs( - selected_names=check_names - ) - dependency_names: frozenset[str] = check_dependency_closure( - graph=python_graph, check_names=check_names - ) - required_loader_names: frozenset[str] = frozenset( - name - for name in dependency_names - if python_graph.nodes_by_name[name].kind == PythonNodeKind.LOADER - ) - required_source_names: frozenset[str] = _required_loader_source_names( - invocation=invocation, - required_loader_names=required_loader_names, - ) - selected_keys: frozenset[CompiledObjectKey] = frozenset( - sql_ref_key(ref) for ref in required_refs - ) | frozenset( - CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=name) - for name in required_source_names - ) - context: StaticCommandContext = compile_static_command_context( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - options=CompilePipelineOptions( - selected_target=request.selected_target, - no_sql_validation=request.no_sql_validation, - connection_config=invocation.connection_config, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ), - selected_keys=selected_keys, - relation_keys=selected_keys, - on_progress=invocation.planning_progress.on_progress, - ) - return build_static_pipeline_result( - context=context, - plan_output=build_relation_command_plan( - project=context.project, - scope=context.scope, - relations=context.relations, - ), - ) - - -def _required_loader_source_names( - *, invocation: CheckInvocation, required_loader_names: frozenset[str] -) -> frozenset[str]: - names: set[str] = set() - for source_file in invocation.discovered_inputs.source_files: - for source in source_file.source_entries: - if source.loader in required_loader_names: - names.add(source.name) - return frozenset(names) diff --git a/src/sqlbuild/cli/commands/main/execution/_check.py b/src/sqlbuild/cli/commands/main/execution/_check.py deleted file mode 100644 index df13f84336..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_check.py +++ /dev/null @@ -1,66 +0,0 @@ -"""CLI check command entry point for Python checks.""" - -from __future__ import annotations - -from sqlbuild._native import write_execution_header -from sqlbuild.cli.commands._helpers.check.execution import ( - execute_check_plan, - prepare_check_execution, -) -from sqlbuild.cli.commands._helpers.check.invocation import resolve_check_invocation -from sqlbuild.cli.commands._helpers.check.outputs import ( - resolve_check_exit_code, - write_check_completion_output, -) -from sqlbuild.cli.commands._helpers.check.planning import compile_check_plan -from sqlbuild.cli.commands.models import ( - CheckCommandRequest, - CheckExecutionPreparation, - CheckInvocation, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult -from sqlbuild.provider.classes.session import ProviderSession -from sqlbuild.provider.main.session import build_provider_session - - -def run_check(request: CheckCommandRequest) -> int: - """Execute the check command.""" - - invocation: CheckInvocation = resolve_check_invocation(request=request) - invocation.progress_stream.write("\n") - write_execution_header( - stream=invocation.progress_stream, - command="sqb check", - target=None, - concurrency=1, - use_color=invocation.use_color, - ) - pipeline_result: CompilePipelineResult = compile_check_plan( - request=request, - invocation=invocation, - ) - preparation: CheckExecutionPreparation = prepare_check_execution( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - ) - provider_session: ProviderSession = build_provider_session( - discovered_providers=invocation.discovered_inputs.providers - ) - try: - results: tuple[PythonCheckExecutionResult, ...] = execute_check_plan( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - providers=provider_session.providers, - ) - write_check_completion_output( - request=request, - invocation=invocation, - preparation=preparation, - results=results, - ) - return resolve_check_exit_code(results) - finally: - provider_session.close() diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 72c900f204..e03115a344 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -57,13 +57,12 @@ CompilerDiagnostic, ) from sqlbuild.compiler.compile.types import CompiledProjectView - from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs + from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ( CompilePipelineResult, PythonPlanEntry, ) from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput - from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan from sqlbuild.compiler.refactoring.models import RefactorProject from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult from sqlbuild.executor.diff.models import DiffExecutionResult, FullDiffSizeLimits @@ -71,7 +70,6 @@ from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.integrations.dbt.models import DbtInitRequest from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter - from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.runtime.contracts.types import NodeStartCallback @@ -301,20 +299,6 @@ class CheckInvocation: planning_progress: PlanningProgressReporter -@dataclass(frozen=True) -class CheckExecutionPreparation: - """Prepared Python graph, check selection, lifecycle, and relation defaults.""" - - python_graph: PythonNodeGraph - check_functions: tuple[DiscoveredCheckFunction, ...] - lifecycle_plan: PythonSqlRunLifecyclePlan - relation_targets: dict[SqlResourceRef, str] - relation_refs: frozenset[SqlResourceRef] - default_database: str | None - default_schema: str | None - project_relations: dict[SqlResourceRef, str] | None = None - - @dataclass(frozen=True) class CloneCommandRequest: """CLI inputs for one clone command invocation.""" diff --git a/src/sqlbuild/compiler/planner/main/commands/relation_plan.py b/src/sqlbuild/compiler/planner/main/commands/_relation_plan.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/commands/relation_plan.py rename to src/sqlbuild/compiler/planner/main/commands/_relation_plan.py diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py index 1f6b7b4de5..e5883bc1d0 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py @@ -3,13 +3,16 @@ from __future__ import annotations import sys +from functools import partial from pathlib import Path import pytest from sqlbuild.cli.commands import models as command_models from sqlbuild.cli.commands._helpers.audit.invocation import resolve_audit_invocation -from sqlbuild.cli.commands._helpers.check.invocation import resolve_check_invocation +from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( + resolve_reported_command_invocation, +) from sqlbuild.cli.commands._helpers.seed.invocation import resolve_seed_invocation from sqlbuild.cli.commands._helpers.test.invocation import resolve_test_invocation from tests.integration.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( @@ -35,7 +38,10 @@ ), CommandInvocationTestCase( description="check text output reports progress on stdout", - resolve=resolve_check_invocation, + resolve=partial( + resolve_reported_command_invocation, + invocation_type=command_models.CheckInvocation, + ), request_type=command_models.CheckCommandRequest, json_output=False, expected_invocation_type_name="CheckInvocation", diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py index dc18c8a95e..5eec82056d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py @@ -4,28 +4,10 @@ from contextvars import Token from io import StringIO -from types import SimpleNamespace -from typing import Any, cast import pytest -import sqlbuild.cli.commands._helpers.check.execution as check_execution -from sqlbuild._native import NativeProgressProjector -from sqlbuild.cli.commands._helpers.check.core import write_check_results -from sqlbuild.cli.commands._helpers.check.execution import execute_check_plan -from sqlbuild.cli.commands.models import CheckExecutionPreparation, CheckInvocation -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.executor.python_nodes.classes.run_state import PythonNodeRunState -from sqlbuild.executor.python_nodes.models import ( - PythonCheckExecutionResult, - PythonIngressLoaderExecutorResult, -) -from sqlbuild.observability import ( - EventDispatcher, - ResourceAttemptLifecycle, - dispatcher_scope, - invocation_scope, -) +from sqlbuild._native import NativeProgressProjector, write_check_results from tests.unit.src.sqlbuild.cli.commands._helpers.check._test_types import ( CheckOutputProjectionCase, ) @@ -42,15 +24,6 @@ def write(self, text: str) -> int: return super().write(text) -class _CloseFailingAdapter: - def connect(self, connection_config: dict[str, object]) -> object: - return connection_config - - def close(self, connection: object) -> None: - del connection - raise OSError("controlled adapter close failure") - - @pytest.mark.parametrize( "test_case", ( @@ -117,95 +90,3 @@ def test_given_two_completed_checks_when_second_row_fails_then_close_renders_unc assert output.count(f"check second OK {test_case.expected_second_duration}") == ( test_case.expected_generic_terminal_count ) - - -@pytest.mark.parametrize( - "test_case", - ( - CheckOutputProjectionCase( - description="adapter close failure leaves all executed check terminals unclaimed", - expected_first_duration="0.00s", - expected_second_duration="0.00s", - expected_generic_terminal_count=2, - ), - ), - ids=lambda case: case.description, -) -def test_given_checks_complete_when_adapter_close_fails_then_close_renders_all_terminals( - monkeypatch: pytest.MonkeyPatch, - test_case: CheckOutputProjectionCase, -) -> None: - stream: StringIO = StringIO() - projector: NativeProgressProjector = NativeProgressProjector(stream=stream, use_color=False) - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=projector.consume, accepts_opaque=False) - run_state: PythonNodeRunState = PythonNodeRunState() - ingress_result: PythonIngressLoaderExecutorResult = PythonIngressLoaderExecutorResult( - python_results=(), load_results=(), run_state=run_state - ) - monkeypatch.setattr(check_execution, "_execute_check_ingress", lambda **_: ingress_result) - monkeypatch.setattr(check_execution, "_execute_check_read_side", lambda **_: ()) - - def execute_checks(**_: Any) -> tuple[PythonCheckExecutionResult, ...]: - with ResourceAttemptLifecycle( - resource_id="check:first", - resource_kind="check", - resource_name="first", - run_id="check-run", - ): - pass - with ResourceAttemptLifecycle( - resource_id="check:second", - resource_kind="check", - resource_name="second", - run_id="check-run", - ): - pass - return check_results() - - monkeypatch.setattr(check_execution, "execute_python_checks", execute_checks) - invocation: CheckInvocation = cast( - CheckInvocation, - SimpleNamespace( - adapter=_CloseFailingAdapter(), - connection_config={}, - use_color=False, - ), - ) - preparation: CheckExecutionPreparation = cast( - CheckExecutionPreparation, - SimpleNamespace( - check_functions=(SimpleNamespace(name="first"), SimpleNamespace(name="second")), - python_graph=object(), - lifecycle_plan=object(), - relation_targets={}, - default_database=None, - default_schema=None, - project_relations=None, - ), - ) - pipeline_result: CompilePipelineResult = cast( - CompilePipelineResult, - SimpleNamespace( - project=SimpleNamespace( - run_id="check-run", effective_target_name=None, effective_vars={} - ), - plan_output=SimpleNamespace(source_map={}), - ), - ) - token: Token[NativeProgressProjector | None] = projector.install() - with invocation_scope("check-invocation"), dispatcher_scope(dispatcher): - with pytest.raises(OSError, match="controlled adapter close failure"): - execute_check_plan( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - providers=cast(Any, None), - ) - projector.restore(token) - projector.close() - output: str = stream.getvalue() - - assert output.count("check first OK") == 1 - assert output.count("check second OK") == 1 - assert output.count(" OK ") == test_case.expected_generic_terminal_count diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index 8c449efd1f..f8245e03e2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -8,7 +8,6 @@ from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.cli.commands._helpers.check import core as check_core -from sqlbuild.cli.commands._helpers.check.core import check_dependency_closure from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation from sqlbuild.compiler.pipeline.models import CompilePipelineResult @@ -23,35 +22,6 @@ ) -@pytest.mark.parametrize( - "test_case", - ( - CheckPlanningTestCase( - description="transitive check dependency closure", - expected_names=frozenset({"asset", "task", "loader"}), - ), - ), - ids=lambda case: case.description, -) -def test_given_check_with_transitive_dependencies_when_planning_then_entire_closure_is_selected( - test_case: CheckPlanningTestCase, -) -> None: - graph: Mock = Mock( - upstream_deps={ - "quality_check": ("asset",), - "asset": ("task", "loader"), - "task": (), - "loader": (), - } - ) - - selected: frozenset[str] = check_dependency_closure( - graph=graph, check_names=frozenset({"quality_check"}) - ) - - assert selected == test_case.expected_names - - @pytest.mark.parametrize( "test_case", ( diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py index 523b555642..321a6da001 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py @@ -11,9 +11,9 @@ from sqlbuild._native import ( ExecutionEventWriter, TerminalEventIndex, + format_check_json, terminal_event_index_scope, ) -from sqlbuild.cli.commands._helpers.check.core import format_check_json from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode From 5f2b053499078ac2ece0dd8c60c3b0cc881de2b0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:33:39 +0100 Subject: [PATCH 20/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 14458477ce..d23870b8e4 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 84966 +remaining_lines = 84185 [[stays]] path = "executor" From 1f071c07e503ee446458abd53ba1d8dad63efec9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:43:27 +0100 Subject: [PATCH 21/37] refactor(cli): render sqb diff output and resolve its exit codes natively --- .../src/diff_report/_helpers/evidence.rs | 110 +++ .../src/diff_report/_helpers/findings.rs | 35 + .../src/diff_report/_helpers/layout.rs | 511 ++++++++++++++ .../src/diff_report/_helpers/mod.rs | 7 + .../src/diff_report/_helpers/numbers.rs | 31 + .../src/diff_report/_helpers/payload.rs | 346 ++++++++++ .../src/diff_report/_helpers/sections.rs | 633 +++++++++++++++++ .../src/diff_report/_helpers/text_cells.rs | 179 +++++ .../sqlbuild-cli/src/diff_report/constants.rs | 10 + .../src/diff_report/main/diff_exit_code.rs | 22 + .../sqlbuild-cli/src/diff_report/main/mod.rs | 4 + .../main/render_diff_error_json.rs | 33 + .../src/diff_report/main/render_diff_json.rs | 19 + .../src/diff_report/main/render_diff_text.rs | 63 ++ crates/sqlbuild-cli/src/diff_report/mod.rs | 10 + crates/sqlbuild-cli/src/diff_report/models.rs | 132 ++++ crates/sqlbuild-cli/src/diff_report/tests.rs | 6 + .../src/diff_report/tests/helpers.rs | 64 ++ .../src/diff_report/tests/test_render.rs | 222 ++++++ .../src/diff_report/tests/test_types.rs | 44 ++ crates/sqlbuild-cli/src/diff_report/types.rs | 9 + crates/sqlbuild-cli/src/lib.rs | 1 + .../src/bindings/_helpers/cli/entry.rs | 3 +- .../src/bindings/_helpers/cli/registration.rs | 5 +- .../bindings/_helpers/diff_report/facts.rs | 273 ++++++++ .../src/bindings/_helpers/diff_report/mod.rs | 2 + .../_helpers/diff_report/registration.rs | 265 +++++++ .../src/bindings/_helpers/mod.rs | 1 + src/sqlbuild/_native.pyi | 48 ++ .../cli/commands/_helpers/diff/evidence.py | 108 --- .../cli/commands/_helpers/diff/execution.py | 44 +- .../cli/commands/_helpers/diff/json_output.py | 314 --------- .../cli/commands/_helpers/diff/output.py | 652 ------------------ .../cli/commands/_helpers/diff/outputs.py | 91 --- src/sqlbuild/cli/commands/constants.py | 1 - .../cli/commands/main/inspection/_diff.py | 37 +- src/sqlbuild/cli/commands/models.py | 19 - .../cli/commands/_helpers/diff/helpers.py | 2 +- .../cli/commands/_helpers/diff/_test_types.py | 14 - .../commands/_helpers/diff/test_evidence.py | 74 -- .../cli/commands/_helpers/diff/test_output.py | 2 +- 41 files changed, 3104 insertions(+), 1342 deletions(-) create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/evidence.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/findings.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/numbers.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/payload.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/sections.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/constants.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/diff_exit_code.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/mod.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/render_diff_error_json.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/render_diff_json.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/render_diff_text.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/mod.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/models.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/tests.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/tests/helpers.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/tests/test_render.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/tests/test_types.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/diff/evidence.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/diff/json_output.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/diff/output.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/diff/outputs.py delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_evidence.py diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/evidence.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/evidence.rs new file mode 100644 index 0000000000..5e8ca37cb2 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/evidence.rs @@ -0,0 +1,110 @@ +//! Bounded, difference-aware rendering of example values and keys. + +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; + +use crate::diff_report::_helpers::numbers::grouped; +use crate::diff_report::constants::MAX_RENDERED_KEY_LENGTH; +use crate::diff_report::models::{DiffExampleOptions, RenderedValue}; + +fn first_difference(left: &[char], right: &[char]) -> Option { + if let Some(index) = left.iter().zip(right).position(|(a, b)| a != b) { + return Some(index); + } + (left.len() != right.len()).then(|| left.len().min(right.len())) +} + +fn render_value( + text: &[char], + first_difference: Option, + options: DiffExampleOptions, +) -> RenderedValue { + let original_length: usize = text.len(); + let full: String = text.iter().collect(); + if options.suppress_values { + return RenderedValue { + text: None, + original_length, + truncated: false, + suppressed: true, + first_difference, + }; + } + let Some(limit) = options + .max_value_length + .filter(|limit| original_length > *limit) + else { + return RenderedValue { + text: Some(full), + original_length, + truncated: false, + suppressed: false, + first_difference, + }; + }; + let difference: usize = first_difference.unwrap_or(0); + let context_before: usize = (limit / 3).min(limit.saturating_sub(1)); + let start: usize = difference + .saturating_sub(context_before) + .min(original_length - limit); + let end: usize = start + limit; + let fragment: String = text[start..end].iter().collect(); + RenderedValue { + text: Some(format!( + "{}{fragment}{}", + if start > 0 { "…" } else { "" }, + if end < original_length { "…" } else { "" } + )), + original_length, + truncated: true, + suppressed: false, + first_difference, + } +} + +/// Two unequal values rendered around their first differing character. +pub(crate) fn render_example_pair( + left: &str, + right: &str, + options: DiffExampleOptions, +) -> (RenderedValue, RenderedValue) { + let left: Vec = left.chars().collect(); + let right: Vec = right.chars().collect(); + let difference: Option = first_difference(&left, &right); + ( + render_value(&left, difference, options), + render_value(&right, difference, options), + ) +} + +/// One bounded value for terminal output. +pub(crate) fn display_example_value(value: &RenderedValue) -> String { + if value.suppressed { + return format!( + "", + grouped(value.original_length as i64) + ); + } + let rendered: &str = value.text.as_deref().unwrap_or_default(); + if value.truncated { + return format!( + "{rendered} [truncated; {} chars]", + grouped(value.original_length as i64) + ); + } + rendered.to_owned() +} + +/// Ordinary keys complete; exceptionally long keys shortened and identified by digest. +pub(crate) fn render_key_value(text: &str) -> String { + let length: usize = text.chars().count(); + if length <= MAX_RENDERED_KEY_LENGTH { + return text.to_owned(); + } + let digest: String = sha256_hex(text.as_bytes()); + let prefix: String = text.chars().take(MAX_RENDERED_KEY_LENGTH).collect(); + format!( + "{prefix}… [truncated; {} chars; sha256:{}]", + grouped(length as i64), + &digest[..12] + ) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/findings.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/findings.rs new file mode 100644 index 0000000000..671dad9ffe --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/findings.rs @@ -0,0 +1,35 @@ +//! Whether a diff found differences, and the resulting outcome. + +use crate::diff_report::constants::{FINDINGS_OUTCOME, PASS_OUTCOME}; +use crate::diff_report::models::DiffModelFacts; + +pub(crate) fn has_schema_differences(model: &DiffModelFacts) -> bool { + !(model.added_columns.is_empty() + && model.removed_columns.is_empty() + && model.type_changes.is_empty()) +} + +/// Whether any model has schema or row differences. +pub(crate) fn has_findings(models: &[DiffModelFacts]) -> bool { + models.iter().any(|model| { + has_schema_differences(model) + || model.rows.as_ref().is_some_and(|rows| { + rows.unequal_count != 0 || rows.left_only_count != 0 || rows.right_only_count != 0 + }) + }) +} + +/// The explicit outcome, or `findings`/`pass` from the results. +pub(crate) fn resolved_outcome(models: &[DiffModelFacts], outcome: Option<&str>) -> String { + outcome + .filter(|outcome| !outcome.is_empty()) + .map(str::to_owned) + .unwrap_or_else(|| { + if has_findings(models) { + FINDINGS_OUTCOME + } else { + PASS_OUTCOME + } + .to_owned() + }) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs new file mode 100644 index 0000000000..c7e36b2e4f --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs @@ -0,0 +1,511 @@ +//! The subset of Rich's console layout the diff summary uses: text, groups, tables and columns. + +use crate::diff_report::_helpers::text_cells::{cell_len, wrap_line}; + +/// Text attributes and one standard foreground color, as Rich renders them. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct Style { + pub(crate) bold: bool, + pub(crate) dim: bool, + pub(crate) italic: bool, + /// The SGR foreground code, such as 36 for cyan; zero for none. + pub(crate) color: u8, +} + +impl Style { + pub(crate) const PLAIN: Self = Self { + bold: false, + dim: false, + italic: false, + color: 0, + }; + + /// This style with `other`'s attributes and color layered on top. + pub(crate) fn with(self, other: Self) -> Self { + Self { + bold: self.bold || other.bold, + dim: self.dim || other.dim, + italic: self.italic || other.italic, + color: if other.color == 0 { + self.color + } else { + other.color + }, + } + } + + fn sgr(self) -> String { + let mut codes: Vec = Vec::new(); + for (enabled, code) in [(self.bold, "1"), (self.dim, "2"), (self.italic, "3")] { + if enabled { + codes.push(code.to_owned()); + } + } + if self.color != 0 { + codes.push(self.color.to_string()); + } + codes.join(";") + } +} + +#[derive(Clone, Debug)] +pub(crate) struct Segment { + text: String, + style: Style, +} + +pub(crate) type Line = Vec; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Justify { + Left, + Center, + Right, +} + +/// Box-drawing characters per line kind, each as left, horizontal, divider and right. +#[derive(Clone, Copy, Debug)] +pub(crate) struct BoxChars { + pub(crate) top: [char; 4], + pub(crate) row: [char; 4], + pub(crate) separator: [char; 4], + pub(crate) bottom: [char; 4], +} + +#[derive(Clone, Debug)] +pub(crate) struct Column { + pub(crate) justify: Justify, + pub(crate) style: Style, + /// Maximum content cells; longer content folds. + pub(crate) max_width: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct Table { + pub(crate) columns: Vec, + pub(crate) rows: Vec>, + /// Whether a section separator follows each row. + pub(crate) section_after: Vec, + pub(crate) borders: Option, + /// Left and right cell padding. + pub(crate) padding: (usize, usize), + pub(crate) collapse_padding: bool, + pub(crate) pad_edge: bool, +} + +#[derive(Clone, Debug)] +pub(crate) enum Node { + Text(String, Style), + Group(Vec), + Table(Table), + Columns(Vec), + /// A heavy-bordered panel spanning the width with one centered title line. + Panel(String, Style), +} + +/// How a table cell justifies and overflows its text. +#[derive(Clone, Copy, Debug)] +struct CellLayout { + justify: Justify, + fold: bool, +} + +impl Node { + pub(crate) fn plain(text: impl Into) -> Self { + Self::Text(text.into(), Style::PLAIN) + } + + pub(crate) fn styled(text: impl Into, style: Style) -> Self { + Self::Text(text.into(), style) + } +} + +impl Table { + /// `Table.grid`: no borders, collapsed padding and no edge padding. + pub(crate) fn grid(columns: Vec, right_padding: usize) -> Self { + Self { + columns, + rows: Vec::new(), + section_after: Vec::new(), + borders: None, + padding: (right_padding, right_padding), + collapse_padding: true, + pad_edge: false, + } + } + + pub(crate) fn add_row(&mut self, cells: Vec) { + self.rows.push(cells); + self.section_after.push(false); + } + + pub(crate) fn add_section(&mut self) { + if let Some(last) = self.section_after.last_mut() { + *last = true; + } + } +} + +pub(crate) fn column(justify: Justify, style: Style) -> Column { + Column { + justify, + style, + max_width: None, + } +} + +fn text_lines(text: &str) -> Vec<&str> { + text.split('\n').collect() +} + +fn segment(text: String, style: Style) -> Segment { + Segment { text, style } +} + +fn line_cells(line: &Line) -> usize { + line.iter().map(|part| cell_len(&part.text)).sum() +} + +/// `Measurement.get(...).maximum` of one node within `max_width` cells. +fn measure(node: &Node, max_width: usize) -> usize { + let natural: usize = match node { + Node::Text(text, _) => text_lines(text) + .into_iter() + .map(cell_len) + .max() + .unwrap_or(0), + Node::Group(children) => children + .iter() + .map(|child| measure(child, max_width)) + .max() + .unwrap_or(0), + Node::Table(table) => { + let extra: usize = table_extra_width(table); + column_widths(table, max_width.saturating_sub(extra)) + .iter() + .sum::() + + extra + + if table.rows.is_empty() { max_width } else { 0 } + } + Node::Columns(_) | Node::Panel(..) => max_width, + }; + natural.min(max_width) +} + +fn table_extra_width(table: &Table) -> usize { + table.borders.map_or(0, |_| table.columns.len() + 1) +} + +/// The left and right padding of one cell column. +fn cell_padding(table: &Table, index: usize) -> (usize, usize) { + let (mut left, mut right) = table.padding; + if table.collapse_padding && index > 0 { + left = left.saturating_sub(right); + } + if !table.pad_edge { + if index == 0 { + left = 0; + } + if index + 1 == table.columns.len() { + right = 0; + } + } + (left, right) +} + +fn column_maximum(table: &Table, index: usize, max_width: usize) -> usize { + let (left, right) = cell_padding(table, index); + let padding: usize = left + right; + let mut widest: Option = None; + for row in &table.rows { + if let Some(cell) = row.get(index) { + let width: usize = measure(cell, max_width.saturating_sub(padding)) + padding; + widest = Some(widest.map_or(width, |current| current.max(width))); + } + } + let mut maximum: usize = widest.unwrap_or(max_width).min(max_width); + if let Some(limit) = table.columns[index].max_width { + let (configured_left, configured_right) = table.padding; + let configured: usize = if table.collapse_padding && index > 0 { + configured_left.saturating_sub(configured_right) + configured_right + } else { + configured_left + configured_right + }; + maximum = maximum.min(limit + configured); + } + maximum +} + +fn ratio_reduce( + total: usize, + ratios: &[usize], + maximums: &[usize], + values: &[usize], +) -> Vec { + let ratios: Vec = ratios + .iter() + .zip(maximums) + .map(|(ratio, maximum)| if *maximum > 0 { *ratio } else { 0 }) + .collect(); + let mut total_ratio: usize = ratios.iter().sum(); + if total_ratio == 0 { + return values.to_vec(); + } + let mut remaining: usize = total; + let mut result: Vec = Vec::with_capacity(values.len()); + for ((ratio, maximum), value) in ratios.iter().zip(maximums).zip(values) { + if *ratio > 0 && total_ratio > 0 { + let share: f64 = (*ratio as f64) * (remaining as f64) / (total_ratio as f64); + let distributed: usize = (*maximum).min(share.round_ties_even() as usize); + result.push(value.saturating_sub(distributed)); + remaining = remaining.saturating_sub(distributed); + total_ratio -= ratio; + } else { + result.push(*value); + } + } + result +} + +fn collapse_widths(mut widths: Vec, max_width: usize) -> Vec { + let mut total: usize = widths.iter().sum(); + while total > 0 && total > max_width { + let excess: usize = total - max_width; + let widest: usize = widths.iter().copied().max().unwrap_or(0); + let second: usize = widths + .iter() + .copied() + .filter(|width| *width != widest) + .max() + .unwrap_or(0); + let difference: usize = widest - second; + if difference == 0 { + break; + } + let ratios: Vec = widths + .iter() + .map(|width| usize::from(*width == widest)) + .collect(); + let maximums: Vec = vec![excess.min(difference); widths.len()]; + widths = ratio_reduce(excess, &ratios, &maximums, &widths); + total = widths.iter().sum(); + } + widths +} + +/// `Table._calculate_column_widths` for a table that does not expand. +fn column_widths(table: &Table, max_width: usize) -> Vec { + let mut widths: Vec = (0..table.columns.len()) + .map(|index| column_maximum(table, index, max_width).max(1)) + .collect(); + if widths.iter().sum::() > max_width { + widths = collapse_widths(widths, max_width); + let total: usize = widths.iter().sum(); + if total > max_width { + let ones: Vec = vec![1; widths.len()]; + widths = ratio_reduce(total - max_width, &ones, &widths.clone(), &widths); + } + widths = widths + .iter() + .enumerate() + .map(|(index, width)| column_maximum(table, index, *width)) + .collect(); + } + widths +} + +fn justified(text: String, width: usize, justify: Justify) -> String { + let content: String = match justify { + Justify::Left => text, + Justify::Center | Justify::Right => text.trim_end().to_owned(), + }; + let used: usize = cell_len(&content); + let free: usize = width.saturating_sub(used); + match justify { + Justify::Left => content + &" ".repeat(free), + Justify::Center => " ".repeat(free / 2) + &content + &" ".repeat(free - free / 2), + Justify::Right => " ".repeat(free) + &content, + } +} + +/// Render a node into lines at most `width` cells wide. +fn render(node: &Node, width: usize, layout: Option, base: Style) -> Vec { + match node { + Node::Text(text, style) => { + let combined: Style = base.with(*style); + let fold: bool = layout.is_none_or(|cell| cell.fold); + let mut lines: Vec = Vec::new(); + for source in text_lines(text) { + for piece in wrap_line(source, width, fold) { + let piece: String = match layout { + Some(cell) => justified(piece, width, cell.justify), + None => piece, + }; + lines.push(vec![segment(piece, combined)]); + } + } + lines + } + Node::Group(children) => children + .iter() + .flat_map(|child| render(child, width, layout, base)) + .collect(), + Node::Table(table) => render_table(table, width), + Node::Panel(title, style) => { + let inner: usize = width.saturating_sub(4); + vec![ + border_line(['┏', '━', '━', '┓'], &[width.saturating_sub(2)]), + vec![ + segment("┃ ".to_owned(), Style::PLAIN), + segment(justified(title.clone(), inner, Justify::Center), *style), + segment(" ┃".to_owned(), Style::PLAIN), + ], + border_line(['┗', '━', '━', '┛'], &[width.saturating_sub(2)]), + ] + } + Node::Columns(items) => render_table(&columns_grid(items, width), width), + } +} + +/// The grid `Columns` lays its items out in. +fn columns_grid(items: &[Node], max_width: usize) -> Table { + let widths: Vec = items.iter().map(|item| measure(item, max_width)).collect(); + let mut count: usize = items.len().max(1); + while count > 1 { + let mut column_widths: Vec = Vec::new(); + let mut overflow: bool = false; + for (position, width) in widths.iter().enumerate() { + let column_index: usize = position % count; + if column_widths.len() <= column_index { + column_widths.push(0); + } + column_widths[column_index] = column_widths[column_index].max(*width); + if column_widths.iter().sum::() > max_width { + count = column_widths.len() - 1; + overflow = true; + break; + } + } + if !overflow { + break; + } + } + let mut grid: Table = Table::grid(vec![column(Justify::Left, Style::PLAIN); count], 0); + for chunk in items.chunks(count) { + let mut cells: Vec = chunk.to_vec(); + cells.resize(count, Node::plain("")); + grid.add_row(cells); + } + grid +} + +fn border_line(chars: [char; 4], widths: &[usize]) -> Line { + let [left, horizontal, divider, right] = chars; + let mut text: String = left.to_string(); + for (index, width) in widths.iter().enumerate() { + if index > 0 { + text.push(divider); + } + text.push_str(&horizontal.to_string().repeat(*width)); + } + text.push(right); + vec![segment(text, Style::PLAIN)] +} + +fn render_cell(table: &Table, index: usize, cell: &Node, width: usize) -> Vec { + let spec: &Column = &table.columns[index]; + let (left, right) = cell_padding(table, index); + let content_width: usize = width.saturating_sub(left + right); + let mut lines: Vec = Vec::new(); + let layout: CellLayout = CellLayout { + justify: spec.justify, + fold: spec.max_width.is_some(), + }; + for mut line in render(cell, content_width, Some(layout), spec.style) { + let used: usize = line_cells(&line); + let mut padded: Line = Vec::new(); + if left > 0 { + padded.push(segment(" ".repeat(left), spec.style)); + } + padded.append(&mut line); + if used < content_width { + padded.push(segment(" ".repeat(content_width - used), spec.style)); + } + if right > 0 { + padded.push(segment(" ".repeat(right), spec.style)); + } + lines.push(padded); + } + lines +} + +fn render_table(table: &Table, max_width: usize) -> Vec { + let extra: usize = table_extra_width(table); + let widths: Vec = column_widths(table, max_width.saturating_sub(extra)); + let mut lines: Vec = Vec::new(); + if let Some(chars) = table.borders { + lines.push(border_line(chars.top, &widths)); + } + for (row_index, row) in table.rows.iter().enumerate() { + let empty: Node = Node::plain(""); + let cells: Vec> = widths + .iter() + .enumerate() + .map(|(index, width)| { + render_cell(table, index, row.get(index).unwrap_or(&empty), *width) + }) + .collect(); + let height: usize = cells.iter().map(Vec::len).max().unwrap_or(0); + for line_index in 0..height { + let mut line: Line = Vec::new(); + if let Some(chars) = table.borders { + line.push(segment(chars.row[0].to_string(), Style::PLAIN)); + } + for (index, cell) in cells.iter().enumerate() { + if let (true, Some(chars)) = (index > 0, table.borders) { + line.push(segment(chars.row[2].to_string(), Style::PLAIN)); + } + match cell.get(line_index) { + Some(content) => line.extend(content.iter().cloned()), + None => line.push(segment( + " ".repeat(widths[index]), + table.columns[index].style, + )), + } + } + if let Some(chars) = table.borders { + line.push(segment(chars.row[3].to_string(), Style::PLAIN)); + } + lines.push(line); + } + let separated: bool = table.section_after[row_index] && row_index + 1 < table.rows.len(); + if let (true, Some(chars)) = (separated, table.borders) { + lines.push(border_line(chars.separator, &widths)); + } + } + if let Some(chars) = table.borders { + lines.push(border_line(chars.bottom, &widths)); + } + lines +} + +/// Render top-level nodes as `Console.print` calls do, one after another. +pub(crate) fn print_nodes(nodes: &[Node], width: usize, use_color: bool) -> String { + let mut output: String = String::new(); + for node in nodes { + for line in render(node, width, None, Style::PLAIN) { + for part in line { + if part.text.is_empty() { + continue; + } + if use_color && part.style != Style::PLAIN { + output.push_str(&format!("\x1b[{}m{}\x1b[0m", part.style.sgr(), part.text)); + } else { + output.push_str(&part.text); + } + } + output.push('\n'); + } + } + output.trim_end().to_owned() +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs new file mode 100644 index 0000000000..f847df459f --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod evidence; +pub(crate) mod findings; +pub(crate) mod layout; +pub(crate) mod numbers; +pub(crate) mod payload; +pub(crate) mod sections; +pub(crate) mod text_cells; diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/numbers.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/numbers.rs new file mode 100644 index 0000000000..249e7a7864 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/numbers.rs @@ -0,0 +1,31 @@ +//! Python's `{:,}` and `{:.2%}` number formats. + +/// An integer with comma thousands separators. +pub(crate) fn grouped(value: i64) -> String { + let digits: String = value.unsigned_abs().to_string(); + let mut result: String = String::new(); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + result.push(','); + } + result.push(digit); + } + if value < 0 { + format!("-{result}") + } else { + result + } +} + +/// A ratio as a percentage with two decimals. +pub(crate) fn percent(ratio: f64) -> String { + format!("{:.2}%", ratio * 100.0) +} + +/// `numerator / denominator` as a percentage, or `0.00%` without a denominator. +pub(crate) fn percentage(numerator: i64, denominator: i64) -> String { + if denominator == 0 { + return "0.00%".to_owned(); + } + percent(numerator as f64 / denominator as f64) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/payload.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/payload.rs new file mode 100644 index 0000000000..c46ae03d65 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/payload.rs @@ -0,0 +1,346 @@ +//! The structured JSON diff document, before serialization. + +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::diff_report::_helpers::evidence::render_example_pair; +use crate::diff_report::_helpers::findings::resolved_outcome; +use crate::diff_report::constants::{FINDINGS_OUTCOME, PASS_OUTCOME, QUERY_INPUT_KIND}; +use crate::diff_report::models::{ + DiffCoverageFacts, DiffDocumentOptions, DiffExampleOptions, DiffModelFacts, DiffValue, + RenderedValue, +}; +use crate::diff_report::types::DiffNamedValue; + +pub(crate) fn integer(value: i64) -> JsonValue { + JsonInteger::parse(&value.to_string()).map_or(JsonValue::Null, JsonValue::Integer) +} + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +pub(crate) fn object(entries: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + entries + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + ) +} + +fn strings(values: &[String]) -> JsonValue { + JsonValue::Array(values.iter().map(|value| text(value)).collect()) +} + +fn optional(value: Option<&DiffValue>) -> JsonValue { + value.map_or(JsonValue::Null, |value| value.json.clone()) +} + +/// `{name: value}` where a repeated name keeps its first position and last value. +fn named_values(values: &[DiffNamedValue]) -> JsonValue { + let mut entries: Vec<(String, JsonValue)> = Vec::new(); + for (name, value) in values { + match entries.iter_mut().find(|(existing, _)| existing == name) { + Some(entry) => entry.1 = value.json.clone(), + None => entries.push((name.clone(), value.json.clone())), + } + } + JsonValue::Object(entries) +} + +fn rendered(value: &RenderedValue) -> JsonValue { + object(vec![ + ("value", value.text.as_deref().map_or(JsonValue::Null, text)), + ("original_length", integer(value.original_length as i64)), + ("truncated", JsonValue::Bool(value.truncated)), + ("suppressed", JsonValue::Bool(value.suppressed)), + ( + "first_difference", + value + .first_difference + .map_or(JsonValue::Null, |index| integer(index as i64)), + ), + ]) +} + +fn coverage(coverage: Option<&DiffCoverageFacts>) -> JsonValue { + coverage.map_or(JsonValue::Null, |coverage| { + object(vec![ + ("row_count", integer(coverage.row_count)), + ("minimum_cursor", optional(coverage.minimum_cursor.as_ref())), + ("maximum_cursor", optional(coverage.maximum_cursor.as_ref())), + ]) + }) +} + +fn examples(model: &DiffModelFacts, options: DiffExampleOptions) -> JsonValue { + let mut values_truncated: bool = false; + let mut unequal_rows: Vec = Vec::new(); + for sample in &model.unequal_samples { + let mut cells: Vec = Vec::new(); + for cell in &sample.changed_cells { + let (left, right) = render_example_pair(&cell.left.text, &cell.right.text, options); + values_truncated = values_truncated || left.truncated || right.truncated; + cells.push(object(vec![ + ("column", text(&cell.name)), + ("from", rendered(&left)), + ("to", rendered(&right)), + ])); + } + unequal_rows.push(object(vec![ + ("key", named_values(&sample.key_values)), + ("changed_cells", JsonValue::Array(cells)), + ])); + } + let truncated_by_count: bool = model.rows.as_ref().is_some_and(|rows| { + rows.unequal_count > model.unequal_samples.len() as i64 + || rows.left_only_count > model.left_only_keys.len() as i64 + || rows.right_only_count > model.right_only_keys.len() as i64 + }); + let keys = |samples: &[Vec]| { + JsonValue::Array(samples.iter().map(|sample| named_values(sample)).collect()) + }; + object(vec![ + ("unequal_rows", JsonValue::Array(unequal_rows)), + ("from_only_keys", keys(&model.left_only_keys)), + ("to_only_keys", keys(&model.right_only_keys)), + ("truncated_by_count", JsonValue::Bool(truncated_by_count)), + ("values_truncated", JsonValue::Bool(values_truncated)), + ( + "values_suppressed", + JsonValue::Bool(options.suppress_values), + ), + ( + "max_value_length", + options + .max_value_length + .map_or(JsonValue::Null, |limit| integer(limit as i64)), + ), + ]) +} + +fn model_payload(model: &DiffModelFacts, options: &DiffDocumentOptions) -> JsonValue { + let type_changes: Vec = model + .type_changes + .iter() + .map(|(name, from_type, to_type)| { + object(vec![ + ("name", text(name)), + ("from_type", text(from_type)), + ("to_type", text(to_type)), + ]) + }) + .collect(); + let mut entries: Vec<(&str, JsonValue)> = vec![ + ("input_kind", text(&model.input_kind)), + ("name", text(&model.name)), + ("from_label", text(&options.from_label)), + ("to_label", text(&options.to_label)), + ("from_relation", text(&model.left_relation)), + ("to_relation", text(&model.right_relation)), + ("unique_key", strings(&model.unique_key)), + ("unkeyed", JsonValue::Bool(model.unkeyed)), + ("excluded_columns", strings(&model.excluded_columns)), + ( + "schema", + object(vec![ + ("added_columns", strings(&model.added_columns)), + ("removed_columns", strings(&model.removed_columns)), + ("type_changed_columns", JsonValue::Array(type_changes)), + ]), + ), + ]; + let Some(rows) = &model.rows else { + entries.push(("comparison_scope", text("schema_only"))); + return object(entries); + }; + let keyed = |value: i64| { + if model.unkeyed { + JsonValue::Null + } else { + integer(value) + } + }; + let changed: Vec<(String, JsonValue)> = rows + .columns + .iter() + .map(|column| (column.name.clone(), integer(column.mismatched_count))) + .collect(); + let mismatches: Vec = rows + .columns + .iter() + .filter(|column| column.mismatched_count > 0) + .map(|column| { + object(vec![ + ("column", text(&column.name)), + ("rows", integer(column.mismatched_count)), + ]) + }) + .collect(); + entries.extend([ + ( + "comparison_scope", + text(if rows.exhaustive { + "exhaustive" + } else { + "sampled" + }), + ), + ( + "coverage", + object(vec![ + ( + "requested", + object(vec![ + ( + "cursor_column", + rows.cursor_column.as_deref().map_or(JsonValue::Null, text), + ), + ("start", optional(rows.cursor_start.as_ref())), + ("end", optional(rows.cursor_end.as_ref())), + ]), + ), + ("from", coverage(rows.left_coverage.as_ref())), + ("to", coverage(rows.right_coverage.as_ref())), + ( + "differs", + JsonValue::Bool(rows.cursor_column.is_some() && rows.coverage_differs), + ), + ]), + ), + ( + "sampling", + object(vec![ + ( + "configured_row_limit", + rows.sampling + .map_or(JsonValue::Null, |(limit, _)| integer(limit)), + ), + ( + "seed", + rows.sampling + .map_or(JsonValue::Null, |(_, seed)| integer(seed)), + ), + ("population_keys", keyed(rows.population_count)), + ("compared_keys", keyed(rows.compared_count)), + ( + "coverage_ratio", + JsonValue::Float(if rows.population_count == 0 { + 1.0 + } else { + rows.compared_count as f64 / rows.population_count as f64 + }), + ), + ("exhaustive", JsonValue::Bool(rows.exhaustive)), + ]), + ), + ( + "rows", + object(vec![ + ("from_count", integer(rows.left_count)), + ("to_count", integer(rows.right_count)), + ( + "joined_count", + integer(rows.equal_count + rows.unequal_count), + ), + ("equal_count", integer(rows.equal_count)), + ("unequal_count", integer(rows.unequal_count)), + ("from_only_count", integer(rows.left_only_count)), + ("to_only_count", integer(rows.right_only_count)), + ("changed_columns", JsonValue::Object(unique(changed))), + ("column_mismatches", JsonValue::Array(mismatches)), + ]), + ), + ("examples", examples(model, options.examples)), + ]); + object(entries) +} + +fn unique(entries: Vec<(String, JsonValue)>) -> Vec<(String, JsonValue)> { + let mut result: Vec<(String, JsonValue)> = Vec::new(); + for (key, value) in entries { + match result.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1 = value, + None => result.push((key, value)), + } + } + result +} + +/// Python's `round(value, 6)`. +fn round_six(value: f64) -> f64 { + format!("{value:.6}").parse::().unwrap_or(value) +} + +/// The complete diff document. +pub(crate) fn document(models: &[DiffModelFacts], options: &DiffDocumentOptions) -> JsonValue { + let payloads: Vec = models + .iter() + .map(|model| model_payload(model, options)) + .collect(); + let outcome: String = resolved_outcome(models, options.outcome.as_deref()); + let status: &str = match outcome.as_str() { + PASS_OUTCOME => "no_differences_found", + FINDINGS_OUTCOME => "differences_found", + other => other, + }; + let query_payloads: Vec = models + .iter() + .zip(&payloads) + .filter(|(model, _)| model.input_kind == QUERY_INPUT_KIND) + .map(|(_, payload)| payload.clone()) + .collect(); + let mut entries: Vec<(&str, JsonValue)> = vec![ + ("schema_version", integer(1)), + ("status", text(status)), + ("outcome", text(&outcome)), + ("from", text(&options.from_label)), + ("to", text(&options.to_label)), + ("models", JsonValue::Array(payloads)), + ]; + if !query_payloads.is_empty() { + entries.push(("input_kind", text(QUERY_INPUT_KIND))); + entries.push(("query_comparisons", JsonValue::Array(query_payloads))); + } + if let Some(phases) = &options.phase_seconds { + let mut sorted: Vec<(String, f64)> = phases.clone(); + sorted.sort_by(|a, b| a.0.cmp(&b.0)); + entries.push(( + "phase_seconds", + JsonValue::Object( + sorted + .into_iter() + .map(|(phase, seconds)| (phase, JsonValue::Float(round_six(seconds)))) + .collect(), + ), + )); + } + json_safe(object(entries)) +} + +/// Non-finite floats as `{"kind": "non_finite_number", "value": ...}` objects. +fn json_safe(value: JsonValue) -> JsonValue { + match value { + JsonValue::Float(number) if !number.is_finite() => { + let label: &str = if number.is_nan() { + "NaN" + } else if number.is_sign_positive() { + "Infinity" + } else { + "-Infinity" + }; + object(vec![ + ("kind", text("non_finite_number")), + ("value", text(label)), + ]) + } + JsonValue::Array(items) => JsonValue::Array(items.into_iter().map(json_safe).collect()), + JsonValue::Object(entries) => JsonValue::Object( + entries + .into_iter() + .map(|(key, item)| (key, json_safe(item))) + .collect(), + ), + other => other, + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/sections.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/sections.rs new file mode 100644 index 0000000000..ba51c25674 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/sections.rs @@ -0,0 +1,633 @@ +//! The sections of the terminal diff summary, as layout nodes. + +use std::collections::BTreeMap; + +use crate::diff_report::_helpers::evidence::{ + display_example_value, render_example_pair, render_key_value, +}; +use crate::diff_report::_helpers::findings::has_schema_differences; +use crate::diff_report::_helpers::layout::{BoxChars, Column, Justify, Node, Style, Table, column}; +use crate::diff_report::_helpers::numbers::{grouped, percent, percentage}; +use crate::diff_report::constants::{ + CHANGED_COLUMN_NAME_WIDTH, QUERY_INPUT_KIND, TOP_CHANGED_COLUMNS, +}; +use crate::diff_report::models::{ + DiffColumnFacts, DiffCoverageFacts, DiffExampleOptions, DiffModelFacts, DiffRowFacts, + DiffTextOptions, +}; +use crate::diff_report::types::DiffNamedValue; + +pub(crate) const BOLD: Style = Style { + bold: true, + ..Style::PLAIN +}; +pub(crate) const OBJECT: Style = Style { + bold: true, + color: 36, + ..Style::PLAIN +}; +pub(crate) const MUTED: Style = Style { + dim: true, + ..Style::PLAIN +}; +const ITALIC: Style = Style { + italic: true, + ..Style::PLAIN +}; +const ADDED: Style = Style { + color: 32, + ..Style::PLAIN +}; +const REMOVED: Style = Style { + color: 31, + ..Style::PLAIN +}; +const WARNING: Style = Style { + color: 33, + ..Style::PLAIN +}; + +const HEAVY_EDGE: BoxChars = BoxChars { + top: ['┏', '━', '┯', '┓'], + row: ['┃', ' ', '│', '┃'], + separator: ['┠', '─', '┼', '┨'], + bottom: ['┗', '━', '┷', '┛'], +}; +const ROUNDED: BoxChars = BoxChars { + top: ['╭', '─', '┬', '╮'], + row: ['│', ' ', '│', '│'], + separator: ['├', '─', '┼', '┤'], + bottom: ['╰', '─', '┴', '╯'], +}; +const SQUARE: BoxChars = BoxChars { + top: ['┌', '─', '┬', '┐'], + row: ['│', ' ', '│', '│'], + separator: ['├', '─', '┼', '┤'], + bottom: ['└', '─', '┴', '┘'], +}; + +/// Everything one model section needs besides the model itself. +pub(crate) struct SectionContext<'a> { + pub(crate) from_label: &'a str, + pub(crate) to_label: &'a str, + pub(crate) text: &'a DiffTextOptions, + pub(crate) examples: DiffExampleOptions, +} + +fn boxed(columns: Vec, borders: BoxChars, padding: (usize, usize)) -> Table { + Table { + columns, + rows: Vec::new(), + section_after: Vec::new(), + borders: Some(borders), + padding, + collapse_padding: false, + pad_edge: true, + } +} + +fn texts(values: &[String]) -> Vec { + values + .iter() + .map(|value| Node::plain(value.clone())) + .collect() +} + +/// The printed nodes of one model: overview and every section. +pub(crate) fn model_nodes(model: &DiffModelFacts, context: &SectionContext<'_>) -> Vec { + let mut nodes: Vec = vec![overview(model, &context.text.mode_label), Node::plain("")]; + nodes.extend(section("Schemas", schema_summary(model))); + let Some(rows) = &model.rows else { + return nodes; + }; + nodes.extend(section("Coverage", coverage(rows, context))); + nodes.extend(section( + if rows.exhaustive { + "Rows" + } else { + "Sampled Rows" + }, + row_counts(rows, context), + )); + nodes.extend(section( + "Changed Columns", + changed_columns(model, rows, context), + )); + let verbose_examples: Option = + if context.text.verbose && !model.unequal_samples.is_empty() { + examples(model, context, None) + } else { + None + }; + if let Some(content) = verbose_examples { + nodes.extend(section("Examples", content)); + } + for (label, samples) in [ + (context.from_label, &model.left_only_keys), + (context.to_label, &model.right_only_keys), + ] { + if !samples.is_empty() { + nodes.extend(section( + &format!("{label} only"), + side_only(label, samples, context.text.max_row_only_examples), + )); + } + } + nodes +} + +fn section(title: &str, content: Node) -> [Node; 4] { + [ + Node::styled(title, BOLD), + Node::styled("▔".repeat(title.chars().count()), BOLD), + content, + Node::plain(""), + ] +} + +fn overview(model: &DiffModelFacts, mode_label: &str) -> Node { + let mut table: Table = Table::grid( + vec![ + column(Justify::Right, OBJECT), + column(Justify::Left, Style::PLAIN), + ], + 2, + ); + let mut add = |label: &str, value: String| { + table.add_row(vec![Node::plain(label), Node::plain(value)]); + }; + add( + if model.input_kind == QUERY_INPUT_KIND { + "Input" + } else { + "Model" + }, + model.name.clone(), + ); + add( + "Key", + if model.unique_key.is_empty() { + "".to_owned() + } else { + model.unique_key.join(", ") + }, + ); + add( + "Comparison", + if model.bounded_fallback { + format!("{mode_label} (fallback to full row diff)") + } else { + mode_label.to_owned() + }, + ); + if model.bounded_fallback { + add( + "Fallback", + "no cursor configured; used full row diff".to_owned(), + ); + } + if !model.excluded_columns.is_empty() { + add("Excluded", model.excluded_columns.join(", ")); + } + if let Some(rows) = &model.rows { + add("Scope", row_scope(rows, model.unkeyed)); + if let Some(label) = tolerance_label(&rows.columns) { + add("Tolerances", label); + } + } + Node::Table(table) +} + +fn tolerance_label(columns: &[DiffColumnFacts]) -> Option { + let labels: Vec = columns + .iter() + .filter_map(|column| { + let (absolute, relative) = column.tolerance.as_ref()?; + let mut parts: Vec = Vec::new(); + if let Some(value) = absolute { + parts.push(format!("absolute={value}")); + } + if let Some(value) = relative { + parts.push(format!("relative={value}")); + } + Some(format!("{} {}", column.name, parts.join(" "))) + }) + .collect(); + (!labels.is_empty()).then(|| labels.join(", ")) +} + +fn row_scope(rows: &DiffRowFacts, unkeyed: bool) -> String { + if unkeyed { + return format!( + "exhaustive ({} multiset rows)", + grouped(rows.population_count) + ); + } + let Some((row_limit, seed)) = rows.sampling else { + return format!("exhaustive ({} union keys)", grouped(rows.population_count)); + }; + if rows.exhaustive { + return format!( + "exhaustive ({} union keys; configured sample limit {})", + grouped(rows.compared_count), + grouped(row_limit) + ); + } + format!( + "sampled {} of {} union keys ({}; seed {seed})", + grouped(rows.compared_count), + grouped(rows.population_count), + percentage(rows.compared_count, rows.population_count) + ) +} + +fn cursor_extent(rows: &DiffRowFacts, coverage: &DiffCoverageFacts) -> String { + if rows.cursor_column.is_none() { + return "".to_owned(); + } + let text = |value: &Option<_>, missing: &str| -> String { + value.as_ref().map_or_else( + || missing.to_owned(), + |value: &crate::diff_report::models::DiffValue| value.text.clone(), + ) + }; + format!( + "{} to {} (requested {} to {})", + text(&coverage.minimum_cursor, ""), + text(&coverage.maximum_cursor, ""), + text(&rows.cursor_start, "unbounded"), + text(&rows.cursor_end, "unbounded") + ) +} + +fn coverage(rows: &DiffRowFacts, context: &SectionContext<'_>) -> Node { + let (Some(left), Some(right)) = (&rows.left_coverage, &rows.right_coverage) else { + return Node::styled("Coverage was not collected.", ITALIC); + }; + let mut table: Table = Table::grid( + vec![ + column(Justify::Left, OBJECT), + column(Justify::Right, Style::PLAIN), + column(Justify::Left, Style::PLAIN), + ], + 2, + ); + table.add_row(texts(&[ + "Side".to_owned(), + "Rows".to_owned(), + "Cursor extent".to_owned(), + ])); + for (label, side) in [(context.from_label, left), (context.to_label, right)] { + table.add_row(texts(&[ + label.to_owned(), + grouped(side.row_count), + cursor_extent(rows, side), + ])); + } + let mut nodes: Vec = vec![Node::Table(table)]; + if rows.cursor_column.is_some() && rows.coverage_differs { + nodes.push(Node::plain("")); + nodes.push(Node::styled( + "Warning: cursor coverage differs between sides; comparison continued without \ + narrowing the requested bounds.", + WARNING, + )); + } + Node::Group(nodes) +} + +fn schema_summary(model: &DiffModelFacts) -> Node { + if !has_schema_differences(model) { + return Node::styled("No schema differences.", ITALIC); + } + let count = |items: usize| grouped(items as i64); + let mut lines: Vec = vec![Node::plain(format!( + "schema differences: {}", + count(model.added_columns.len() + model.removed_columns.len() + model.type_changes.len()) + ))]; + for (label, items) in [ + ("added columns", model.added_columns.len()), + ("removed columns", model.removed_columns.len()), + ("type changes", model.type_changes.len()), + ] { + if items > 0 { + lines.push(Node::plain(format!("{label}: {}", count(items)))); + } + } + Node::Group(lines) +} + +fn delta_label(left: i64, right: i64) -> String { + if left == 0 { + return "(no baseline)".to_owned(); + } + if left == right { + return "(no change)".to_owned(); + } + let ratio: f64 = right as f64 / left as f64; + if right > left { + format!("(+{})", percent(ratio - 1.0)) + } else { + format!("(-{})", percent(1.0 - ratio)) + } +} + +fn venn_box(rows: &[(Node, bool)]) -> Node { + let mut table: Table = boxed( + vec![column(Justify::Left, Style::PLAIN); 5], + HEAVY_EDGE, + (0, 0), + ); + for (cell, section_after) in rows { + table.add_row(vec![cell.clone(); 5]); + if *section_after { + table.add_section(); + } + } + Node::Table(table) +} + +fn row_counts(rows: &DiffRowFacts, context: &SectionContext<'_>) -> Node { + let center: Column = column(Justify::Center, Style::PLAIN); + let mut counts: Table = Table::grid(vec![center.clone(), center.clone(), center], 2); + counts.add_row(texts(&[ + format!("{} count", context.from_label), + String::new(), + format!("{} count", context.to_label), + ])); + counts.add_row(texts(&[ + grouped(rows.left_count), + delta_label(rows.left_count, rows.right_count), + grouped(rows.right_count), + ])); + let joined: i64 = rows.equal_count + rows.unequal_count; + let blank: Node = Node::plain(" "); + let mut left_rows: Vec<(Node, bool)> = Vec::new(); + let mut right_rows: Vec<(Node, bool)> = Vec::new(); + if rows.left_only_count > 0 { + left_rows.push((Node::styled("-", REMOVED), true)); + } + if rows.equal_count > 0 { + left_rows.push((blank.clone(), true)); + right_rows.push((blank.clone(), true)); + } + if rows.unequal_count > 0 { + left_rows.push((blank.clone(), false)); + right_rows.push((blank, true)); + } + if rows.right_only_count > 0 { + right_rows.push((Node::styled("+", ADDED), false)); + } + let shifted: bool = rows.left_only_count > 0; + let right_box: Node = venn_box(&right_rows); + let diagram: Vec = vec![ + venn_box(&left_rows), + row_separator(rows), + if shifted { + Node::Group(vec![Node::plain("\n"), right_box]) + } else { + right_box + }, + row_stats(rows, joined, context), + joined_annotation(rows, joined), + ]; + Node::Group(vec![ + Node::Table(counts), + Node::plain(""), + Node::Columns(diagram), + Node::plain(""), + Node::plain(format!("joined: {}", grouped(joined))), + ]) +} + +fn row_separator(rows: &DiffRowFacts) -> Node { + let mut parts: Vec = Vec::new(); + if rows.left_only_count > 0 { + parts.push(Node::plain("\n")); + } + for (count, symbol) in [(rows.equal_count, " = "), (rows.unequal_count, " ≠ ")] { + if count > 0 { + parts.push(Node::plain("╌╌╌")); + parts.push(Node::styled(symbol, BOLD)); + } + } + parts.push(Node::plain("╌╌╌")); + Node::Group(parts) +} + +fn row_stats(rows: &DiffRowFacts, joined: i64, context: &SectionContext<'_>) -> Node { + let mut table: Table = boxed( + vec![ + column(Justify::Right, Style::PLAIN), + column(Justify::Left, Style::PLAIN), + column(Justify::Right, Style::PLAIN), + ], + ROUNDED, + (1, 0), + ); + let entries: [(i64, String, i64); 4] = [ + ( + rows.left_only_count, + format!("{} only", context.from_label), + rows.left_count, + ), + (rows.equal_count, "equal".to_owned(), joined), + (rows.unequal_count, "unequal".to_owned(), joined), + ( + rows.right_only_count, + format!("{} only", context.to_label), + rows.right_count, + ), + ]; + for (count, label, denominator) in entries { + table.add_row(texts(&[ + grouped(count), + label, + format!("({})", percentage(count, denominator)), + ])); + table.add_section(); + } + Node::Table(table) +} + +fn joined_annotation(rows: &DiffRowFacts, joined: i64) -> Node { + if joined == 0 { + return Node::plain(""); + } + let mut lines: Vec = Vec::new(); + if rows.left_only_count > 0 { + lines.push(Node::plain("\n")); + } + lines.push(Node::plain("╌╮")); + lines.push(Node::plain(" │")); + if rows.equal_count > 0 && rows.unequal_count > 0 { + lines.push(Node::plain(format!("╌├╴ {} joined", grouped(joined)))); + lines.push(Node::plain(" │")); + } + lines.push(Node::plain("╌╯")); + Node::Group(lines) +} + +fn changed_columns( + model: &DiffModelFacts, + rows: &DiffRowFacts, + context: &SectionContext<'_>, +) -> Node { + let mut mismatched: Vec<&DiffColumnFacts> = rows + .columns + .iter() + .filter(|column| column.mismatched_count > 0) + .collect(); + mismatched.sort_by(|a, b| { + b.mismatched_count + .cmp(&a.mismatched_count) + .then_with(|| a.name.cmp(&b.name)) + }); + if mismatched.is_empty() { + return Node::styled( + if rows.exhaustive { + "No changed columns." + } else { + "No changed columns in sampled keys." + }, + ITALIC, + ); + } + let mut name_column: Column = column(Justify::Left, OBJECT); + name_column.max_width = Some(CHANGED_COLUMN_NAME_WIDTH); + let mut table: Table = boxed( + vec![ + name_column, + column(Justify::Right, Style::PLAIN), + column(Justify::Right, Style::PLAIN), + ], + SQUARE, + (1, 1), + ); + let visible: Vec<&DiffColumnFacts> = mismatched + .iter() + .copied() + .take(TOP_CHANGED_COLUMNS) + .collect(); + for item in &visible { + let matching: i64 = (rows.joined_count - item.mismatched_count).max(0); + table.add_row(texts(&[ + item.name.clone(), + format!("mismatches={}", grouped(item.mismatched_count)), + format!("match={}", percentage(matching, rows.joined_count)), + ])); + } + if mismatched.len() > visible.len() { + table.add_section(); + table.add_row(vec![ + Node::styled("...", MUTED), + Node::styled( + format!( + "and {} more", + grouped((mismatched.len() - visible.len()) as i64) + ), + MUTED, + ), + Node::styled("", MUTED), + ]); + } + if context.text.verbose { + return Node::Table(table); + } + let names: Vec = visible.iter().map(|item| item.name.clone()).collect(); + match examples(model, context, Some(&names)) { + Some(content) => Node::Group(vec![ + Node::Table(table), + Node::plain(""), + content, + Node::plain(""), + Node::styled("Use --verbose to show more example row changes.", MUTED), + ]), + None => Node::Table(table), + } +} + +fn key_label(values: &[DiffNamedValue], separator: &str) -> String { + values + .iter() + .map(|(name, value)| format!("{name}={}", render_key_value(&value.text))) + .collect::>() + .join(separator) +} + +fn examples( + model: &DiffModelFacts, + context: &SectionContext<'_>, + visible: Option<&[String]>, +) -> Option { + let mut grouped_examples: BTreeMap<&str, Vec> = BTreeMap::new(); + for sample in &model.unequal_samples { + let key: String = key_label(&sample.key_values, ", "); + for cell in &sample.changed_cells { + let (left, right) = + render_example_pair(&cell.left.text, &cell.right.text, context.examples); + let difference: String = match left.first_difference { + Some(index) if left.truncated || right.truncated => { + format!( + " (first difference at character {})", + grouped(index as i64 + 1) + ) + } + _ => String::new(), + }; + grouped_examples + .entry(cell.name.as_str()) + .or_default() + .push(format!( + "{key} | {} -> {}{difference}", + display_example_value(&left), + display_example_value(&right) + )); + } + } + let limit: usize = context.text.max_column_examples; + let mut blocks: Vec = Vec::new(); + for (name, all) in grouped_examples { + if visible.is_some_and(|names| !names.iter().any(|visible| visible == name)) { + continue; + } + blocks.push(Node::styled(name, OBJECT)); + for example in all.iter().take(limit) { + blocks.push(Node::plain(format!(" - {example}"))); + } + if all.len() > limit { + blocks.push(Node::styled( + format!( + " showing {} of {} examples", + grouped(limit as i64), + grouped(all.len() as i64) + ), + MUTED, + )); + } + blocks.push(Node::plain("")); + } + blocks.pop().map(|_| Node::Group(blocks)) +} + +fn side_only(label: &str, samples: &[Vec], limit: usize) -> Node { + let mut blocks: Vec = samples + .iter() + .take(limit) + .map(|sample| Node::plain(format!(" - {}", key_label(sample, " | ")))) + .collect(); + if samples.len() > limit { + blocks.push(Node::styled( + format!( + " showing {} of {} {label} only rows", + grouped(limit as i64), + grouped(samples.len() as i64) + ), + MUTED, + )); + } + if blocks.is_empty() { + return Node::styled("No side-only samples collected.", ITALIC); + } + Node::Group(blocks) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs new file mode 100644 index 0000000000..6c87a93a03 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs @@ -0,0 +1,179 @@ +//! Terminal cell widths and Rich's word wrapping of one line of text. + +/// The terminal cells one character occupies: zero for combining marks, two for wide characters. +pub(crate) fn char_cells(character: char) -> usize { + let code: u32 = u32::from(character); + let zero_width: bool = matches!( + code, + 0x0300..=0x036F | 0x200B..=0x200F | 0x20D0..=0x20FF | 0xFE00..=0xFE0F | 0xFE20..=0xFE2F + ); + let wide: bool = matches!( + code, + 0x1100..=0x115F + | 0x2E80..=0x303E + | 0x3041..=0x33FF + | 0x3400..=0x4DBF + | 0x4E00..=0x9FFF + | 0xA000..=0xA4CF + | 0xAC00..=0xD7A3 + | 0xF900..=0xFAFF + | 0xFE30..=0xFE4F + | 0xFF00..=0xFF60 + | 0xFFE0..=0xFFE6 + | 0x1F300..=0x1F64F + | 0x1F900..=0x1F9FF + | 0x20000..=0x3FFFD + ); + if zero_width { + 0 + } else if wide { + 2 + } else { + 1 + } +} + +/// The terminal cells a string occupies. +pub(crate) fn cell_len(text: &str) -> usize { + text.chars().map(char_cells).sum() +} + +/// The longest prefix of `text` that fits `width` cells, padded with spaces to exactly `width`. +pub(crate) fn set_cell_size(text: &str, width: usize) -> String { + let mut result: String = String::new(); + let mut used: usize = 0; + for character in text.chars() { + let cells: usize = char_cells(character); + if used + cells > width { + break; + } + used += cells; + result.push(character); + } + result.push_str(&" ".repeat(width - used)); + result +} + +/// Whitespace-then-word-then-whitespace pieces, as `\s*\S+\s*` matches them. +fn words(characters: &[char]) -> Vec<(usize, usize)> { + let mut pieces: Vec<(usize, usize)> = Vec::new(); + let mut position: usize = 0; + while position < characters.len() { + let start: usize = position; + let mut cursor: usize = position; + while cursor < characters.len() && characters[cursor].is_whitespace() { + cursor += 1; + } + if cursor == characters.len() { + break; + } + while cursor < characters.len() && !characters[cursor].is_whitespace() { + cursor += 1; + } + while cursor < characters.len() && characters[cursor].is_whitespace() { + cursor += 1; + } + pieces.push((start, cursor)); + position = cursor; + } + pieces +} + +fn text_cells(characters: &[char]) -> usize { + characters.iter().copied().map(char_cells).sum() +} + +fn trimmed_end(characters: &[char]) -> &[char] { + let mut end: usize = characters.len(); + while end > 0 && characters[end - 1].is_whitespace() { + end -= 1; + } + &characters[..end] +} + +/// `chop_cells`: consecutive pieces of at most `width` cells. +fn chop_cells(characters: &[char], width: usize) -> Vec> { + let mut lines: Vec> = Vec::new(); + let mut line: Vec = Vec::new(); + let mut size: usize = 0; + for character in characters { + let cells: usize = char_cells(*character); + if size + cells > width && !line.is_empty() { + lines.push(std::mem::take(&mut line)); + size = 0; + } + size += cells; + line.push(*character); + } + if !line.is_empty() { + lines.push(line); + } + lines +} + +/// Rich's `divide_line`: character offsets where a line breaks to fit `width` cells. +fn divide_line(characters: &[char], width: usize, fold: bool) -> Vec { + let mut breaks: Vec = Vec::new(); + let mut offset: usize = 0; + for (word_start, word_end) in words(characters) { + let mut start: usize = word_start; + let word: &[char] = &characters[word_start..word_end]; + let word_length: usize = text_cells(trimmed_end(word)); + if width.saturating_sub(offset) >= word_length && offset <= width { + offset += text_cells(word); + } else if word_length > width { + if fold { + let folded: Vec> = chop_cells(word, width); + let count: usize = folded.len(); + for (index, piece) in folded.iter().enumerate() { + if start > 0 { + breaks.push(start); + } + if index + 1 == count { + offset = text_cells(piece); + } else { + start += piece.len(); + } + } + } else { + if start > 0 { + breaks.push(start); + } + offset = text_cells(word); + } + } else if offset > 0 && start > 0 { + breaks.push(start); + offset = text_cells(word); + } + } + breaks +} + +/// One source line wrapped to `width` cells like `Text.wrap` without justification. +pub(crate) fn wrap_line(line: &str, width: usize, fold: bool) -> Vec { + let characters: Vec = line.chars().collect(); + let mut pieces: Vec = Vec::new(); + let mut previous: usize = 0; + let mut breaks: Vec = divide_line(&characters, width, fold); + breaks.push(characters.len()); + for offset in breaks { + let mut piece: Vec = characters[previous..offset].to_vec(); + previous = offset; + if piece.len() > width { + let excess: usize = piece.len() - width; + let trailing: usize = piece.len() - trimmed_end(&piece).len(); + piece.truncate(piece.len() - trailing.min(excess)); + } + let text: String = piece.into_iter().collect(); + pieces.push(if cell_len(&text) > width { + if fold { + set_cell_size(&text, width) + } else { + set_cell_size(&text, width.saturating_sub(1)) + "…" + } + } else { + text + }); + } + pieces +} diff --git a/crates/sqlbuild-cli/src/diff_report/constants.rs b/crates/sqlbuild-cli/src/diff_report/constants.rs new file mode 100644 index 0000000000..a67d7a65b3 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/constants.rs @@ -0,0 +1,10 @@ +//! Fixed widths and limits of the diff summary. + +pub(crate) const SUMMARY_WIDTH: usize = 110; +pub(crate) const TOP_CHANGED_COLUMNS: usize = 5; +pub(crate) const MAX_RENDERED_KEY_LENGTH: usize = 256; +pub(crate) const CHANGED_COLUMN_NAME_WIDTH: usize = 32; +pub(crate) const QUERY_INPUT_KIND: &str = "query"; +pub(crate) const FINDINGS_OUTCOME: &str = "findings"; +pub(crate) const PASS_OUTCOME: &str = "pass"; +pub(crate) const INCOMPLETE_OUTCOME: &str = "incomplete"; diff --git a/crates/sqlbuild-cli/src/diff_report/main/diff_exit_code.rs b/crates/sqlbuild-cli/src/diff_report/main/diff_exit_code.rs new file mode 100644 index 0000000000..e980677d3b --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/diff_exit_code.rs @@ -0,0 +1,22 @@ +//! The outcome and exit code of a finished diff. + +use crate::diff_report::_helpers::findings::{has_findings, has_schema_differences}; +use crate::diff_report::constants::{FINDINGS_OUTCOME, INCOMPLETE_OUTCOME, PASS_OUTCOME}; +use crate::diff_report::models::DiffModelFacts; + +/// `pass` (0), `findings` (1), or `incomplete` (2) when a query schema blocked row comparison. +#[must_use] +pub fn diff_exit_code(models: &[DiffModelFacts], query_row_comparison: bool) -> (String, i32) { + let incomplete: bool = query_row_comparison + && models + .iter() + .any(|model| model.rows.is_none() && has_schema_differences(model)); + let (outcome, code) = if incomplete { + (INCOMPLETE_OUTCOME, 2) + } else if has_findings(models) { + (FINDINGS_OUTCOME, 1) + } else { + (PASS_OUTCOME, 0) + }; + (outcome.to_owned(), code) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/mod.rs b/crates/sqlbuild-cli/src/diff_report/main/mod.rs new file mode 100644 index 0000000000..86714c78a4 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/mod.rs @@ -0,0 +1,4 @@ +pub mod diff_exit_code; +pub mod render_diff_error_json; +pub mod render_diff_json; +pub mod render_diff_text; diff --git a/crates/sqlbuild-cli/src/diff_report/main/render_diff_error_json.rs b/crates/sqlbuild-cli/src/diff_report/main/render_diff_error_json.rs new file mode 100644 index 0000000000..1280a07b75 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/render_diff_error_json.rs @@ -0,0 +1,33 @@ +//! The machine-readable document of an incomplete or failed diff. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::diff_report::_helpers::payload::{integer, object, text}; +use crate::diff_report::models::DiffErrorFacts; + +/// The error document text, without a trailing newline. +pub fn render_diff_error_json(facts: &DiffErrorFacts) -> Result { + let mut error: Vec<(&str, JsonValue)> = vec![ + ("code", text(&facts.code)), + ("message", text(&facts.message)), + ]; + if let Some(help) = &facts.help { + error.push(("help", text(help))); + } + let mut entries: Vec<(&str, JsonValue)> = vec![ + ("schema_version", integer(1)), + ("status", text(&facts.status)), + ("outcome", text(&facts.status)), + ("error", object(error)), + ("models", JsonValue::Array(Vec::new())), + ("query_comparisons", JsonValue::Array(Vec::new())), + ]; + if let Some(size_guard) = &facts.size_guard { + entries.push(("size_guard", size_guard.clone())); + } + let mut settings: StdlibJsonOptions = StdlibJsonOptions::new(Some(2)); + settings.sort_keys = true; + dumps(&object(entries), &JsonDialect::Stdlib(settings)) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/render_diff_json.rs b/crates/sqlbuild-cli/src/diff_report/main/render_diff_json.rs new file mode 100644 index 0000000000..84e56dd2b3 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/render_diff_json.rs @@ -0,0 +1,19 @@ +//! The structured diff document, serialized as `json.dumps(indent=2, sort_keys=True)`. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; + +use crate::diff_report::_helpers::payload::document; +use crate::diff_report::models::{DiffDocumentOptions, DiffModelFacts}; + +/// The diff document text, without a trailing newline. +pub fn render_diff_json( + models: &[DiffModelFacts], + options: &DiffDocumentOptions, +) -> Result { + let mut settings: StdlibJsonOptions = StdlibJsonOptions::new(Some(2)); + settings.sort_keys = true; + settings.allow_nan = false; + dumps(&document(models, options), &JsonDialect::Stdlib(settings)) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/render_diff_text.rs b/crates/sqlbuild-cli/src/diff_report/main/render_diff_text.rs new file mode 100644 index 0000000000..43a4480e2e --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/render_diff_text.rs @@ -0,0 +1,63 @@ +//! The terminal diff summary. + +use crate::diff_report::_helpers::findings::resolved_outcome; +use crate::diff_report::_helpers::layout::{Node, print_nodes}; +use crate::diff_report::_helpers::numbers::grouped; +use crate::diff_report::_helpers::sections::{BOLD, MUTED, OBJECT, SectionContext, model_nodes}; +use crate::diff_report::constants::{QUERY_INPUT_KIND, SUMMARY_WIDTH}; +use crate::diff_report::models::{DiffDocumentOptions, DiffModelFacts, DiffTextOptions}; + +/// Render the summary panel, then every model's overview and sections. +#[must_use] +pub fn render_diff_text( + models: &[DiffModelFacts], + document: &DiffDocumentOptions, + options: &DiffTextOptions, +) -> String { + if models.is_empty() { + return print_nodes( + &[Node::plain("No models selected for diff.")], + SUMMARY_WIDTH, + options.use_color, + ); + } + let input_label: &str = if models + .iter() + .all(|model| model.input_kind == QUERY_INPUT_KIND) + { + "query pairs" + } else { + "models" + }; + let mut nodes: Vec = vec![ + Node::Panel("SQLBuild Diff Summary".to_owned(), BOLD), + Node::styled( + format!("{} vs {}", document.from_label, document.to_label), + OBJECT, + ), + Node::plain(format!( + "selected {input_label}: {}", + grouped(models.len() as i64) + )), + Node::plain(format!( + "outcome: {}", + resolved_outcome(models, document.outcome.as_deref()) + )), + Node::plain(""), + ]; + let context: SectionContext<'_> = SectionContext { + from_label: &document.from_label, + to_label: &document.to_label, + text: options, + examples: document.examples, + }; + for (index, model) in models.iter().enumerate() { + if index > 0 { + nodes.push(Node::plain("")); + nodes.push(Node::styled("─".repeat(SUMMARY_WIDTH), MUTED)); + nodes.push(Node::plain("")); + } + nodes.extend(model_nodes(model, &context)); + } + print_nodes(&nodes, SUMMARY_WIDTH, options.use_color) +} diff --git a/crates/sqlbuild-cli/src/diff_report/mod.rs b/crates/sqlbuild-cli/src/diff_report/mod.rs new file mode 100644 index 0000000000..5f9d1eb986 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/mod.rs @@ -0,0 +1,10 @@ +//! The `sqb diff` summary and JSON documents rendered from plain diff facts. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cli/src/diff_report/models.rs b/crates/sqlbuild-cli/src/diff_report/models.rs new file mode 100644 index 0000000000..10ddab3fd8 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/models.rs @@ -0,0 +1,132 @@ +//! Plain facts of one diff result and the options that render it. + +use sqlbuild_core::json::models::JsonValue; + +use crate::diff_report::types::{DiffNamedValue, DiffTypeChange}; + +/// One fetched warehouse value: its Python `str()` text and its JSON document form. +#[derive(Clone, Debug, PartialEq)] +pub struct DiffValue { + pub text: String, + pub json: JsonValue, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffColumnFacts { + pub name: String, + pub mismatched_count: i64, + /// `str()` of the absolute and relative tolerance, when one is configured. + pub tolerance: Option<(Option, Option)>, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffCoverageFacts { + pub row_count: i64, + pub minimum_cursor: Option, + pub maximum_cursor: Option, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffRowFacts { + pub left_count: i64, + pub right_count: i64, + pub joined_count: i64, + pub equal_count: i64, + pub unequal_count: i64, + pub left_only_count: i64, + pub right_only_count: i64, + pub columns: Vec, + pub population_count: i64, + pub compared_count: i64, + /// The configured sample row limit and seed. + pub sampling: Option<(i64, i64)>, + pub left_coverage: Option, + pub right_coverage: Option, + pub cursor_column: Option, + pub cursor_start: Option, + pub cursor_end: Option, + pub exhaustive: bool, + /// Whether both sides report different cursor extents, by Python value equality. + pub coverage_differs: bool, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct DiffChangedCell { + pub name: String, + pub left: DiffValue, + pub right: DiffValue, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffSampleRow { + pub key_values: Vec, + pub changed_cells: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffModelFacts { + pub name: String, + pub left_relation: String, + pub right_relation: String, + pub added_columns: Vec, + pub removed_columns: Vec, + pub type_changes: Vec, + pub unique_key: Vec, + pub rows: Option, + pub unequal_samples: Vec, + pub left_only_keys: Vec>, + pub right_only_keys: Vec>, + pub bounded_fallback: bool, + pub excluded_columns: Vec, + pub input_kind: String, + pub unkeyed: bool, +} + +/// How example values render: an optional per-value length bound, or suppressed entirely. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DiffExampleOptions { + pub max_value_length: Option, + pub suppress_values: bool, +} + +/// Everything a rendered diff document shows besides the per-model facts. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffDocumentOptions { + pub from_label: String, + pub to_label: String, + pub examples: DiffExampleOptions, + /// An explicit outcome, such as `incomplete`; otherwise derived from the findings. + pub outcome: Option, + /// Per-phase durations of a query diff, in seconds. + pub phase_seconds: Option>, +} + +/// The terminal summary's presentation options. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DiffTextOptions { + pub mode_label: String, + pub use_color: bool, + pub verbose: bool, + pub max_column_examples: usize, + pub max_row_only_examples: usize, +} + +/// A failed or incomplete diff reported as JSON. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffErrorFacts { + pub status: String, + pub code: String, + pub message: String, + pub help: Option, + pub size_guard: Option, +} + +/// One value rendered for display: bounded around the first difference, or suppressed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct RenderedValue { + pub(crate) text: Option, + pub(crate) original_length: usize, + pub(crate) truncated: bool, + pub(crate) suppressed: bool, + pub(crate) first_difference: Option, +} diff --git a/crates/sqlbuild-cli/src/diff_report/tests.rs b/crates/sqlbuild-cli/src/diff_report/tests.rs new file mode 100644 index 0000000000..d2b8e61c63 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_render.rs"] +mod render; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/src/diff_report/tests/helpers.rs b/crates/sqlbuild-cli/src/diff_report/tests/helpers.rs new file mode 100644 index 0000000000..7c8a9c812e --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/tests/helpers.rs @@ -0,0 +1,64 @@ +use sqlbuild_core::json::models::JsonValue; + +use crate::diff_report::models::{ + DiffColumnFacts, DiffDocumentOptions, DiffExampleOptions, DiffModelFacts, DiffRowFacts, + DiffValue, +}; + +pub(crate) fn changed_rows(unequal: i64) -> DiffRowFacts { + DiffRowFacts { + left_count: 3, + right_count: 3, + joined_count: 3, + equal_count: 3 - unequal, + unequal_count: unequal, + columns: vec![DiffColumnFacts { + name: "amount".to_owned(), + mismatched_count: unequal, + tolerance: None, + }], + population_count: 3, + compared_count: 3, + exhaustive: true, + ..DiffRowFacts::default() + } +} + +pub(crate) fn model(rows: Option, added_columns: &[&str]) -> DiffModelFacts { + DiffModelFacts { + name: "orders".to_owned(), + left_relation: "prod.orders".to_owned(), + right_relation: "dev.orders".to_owned(), + added_columns: added_columns + .iter() + .map(|name| (*name).to_owned()) + .collect(), + unique_key: vec!["order_id".to_owned()], + rows, + input_kind: "model".to_owned(), + ..DiffModelFacts::default() + } +} + +pub(crate) fn document() -> DiffDocumentOptions { + DiffDocumentOptions { + from_label: "prod".to_owned(), + to_label: "dev".to_owned(), + examples: DiffExampleOptions { + max_value_length: Some(160), + suppress_values: false, + }, + ..DiffDocumentOptions::default() + } +} + +pub(crate) fn cursor_rows() -> DiffRowFacts { + DiffRowFacts { + cursor_column: Some("ts".to_owned()), + cursor_start: Some(DiffValue { + text: "nan".to_owned(), + json: JsonValue::Float(f64::NAN), + }), + ..changed_rows(1) + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/tests/test_render.rs b/crates/sqlbuild-cli/src/diff_report/tests/test_render.rs new file mode 100644 index 0000000000..6f85c27046 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/tests/test_render.rs @@ -0,0 +1,222 @@ +use crate::diff_report::_helpers::evidence::{ + display_example_value, render_example_pair, render_key_value, +}; +use crate::diff_report::main::diff_exit_code::diff_exit_code; +use crate::diff_report::main::render_diff_error_json::render_diff_error_json; +use crate::diff_report::main::render_diff_json::render_diff_json; +use crate::diff_report::main::render_diff_text::render_diff_text; +use crate::diff_report::models::{DiffErrorFacts, DiffExampleOptions, DiffTextOptions}; +use crate::diff_report::tests::helpers::{changed_rows, cursor_rows, document, model}; +use crate::diff_report::tests::test_types::{ + EvidenceTestCase, JsonTestCase, KeyTestCase, OutcomeTestCase, SummaryTestCase, +}; + +#[test] +fn given_unequal_values_when_rendering_evidence_then_bounds_are_explicit() { + let long = |tail: &str| format!("{}{tail}", "a".repeat(200)); + let test_cases = [ + EvidenceTestCase { + description: "late text difference keeps nearby context", + left: long("left-tail"), + right: long("right-tail"), + options: DiffExampleOptions { + max_value_length: Some(30), + suppress_values: false, + }, + expected_left: "…aaaaaaaaaaaaaaaaaaaaaleft-tail [truncated; 209 chars]".to_owned(), + expected_right: "…aaaaaaaaaaaaaaaaaaaaright-tail [truncated; 210 chars]".to_owned(), + expected_first_difference: Some(200), + }, + EvidenceTestCase { + description: "suppression retains lengths without values", + left: "left-private-value".to_owned(), + right: "right-private-value".to_owned(), + options: DiffExampleOptions { + max_value_length: Some(30), + suppress_values: true, + }, + expected_left: "".to_owned(), + expected_right: "".to_owned(), + expected_first_difference: Some(0), + }, + EvidenceTestCase { + description: "explicit unlimited rendering preserves complete values", + left: long("left-tail"), + right: long("right-tail"), + options: DiffExampleOptions { + max_value_length: None, + suppress_values: false, + }, + expected_left: long("left-tail"), + expected_right: long("right-tail"), + expected_first_difference: Some(200), + }, + ]; + + for test_case in test_cases { + let (left, right) = + render_example_pair(&test_case.left, &test_case.right, test_case.options); + + assert_eq!( + ( + display_example_value(&left), + display_example_value(&right), + left.first_difference, + ), + ( + test_case.expected_left, + test_case.expected_right, + test_case.expected_first_difference, + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_keys_when_rendering_then_long_keys_are_shortened_with_their_digest() { + let test_cases = [ + KeyTestCase { + description: "ordinary key stays complete", + key: "k".repeat(256), + expected_prefix: "k".repeat(256), + expected_suffix: String::new(), + }, + KeyTestCase { + description: "long key names its length and digest", + key: "k".repeat(300), + expected_prefix: format!("{}… [truncated; 300 chars; sha256:", "k".repeat(256)), + expected_suffix: "]".to_owned(), + }, + ]; + + for test_case in test_cases { + let rendered: String = render_key_value(&test_case.key); + + assert_eq!( + ( + rendered.starts_with(&test_case.expected_prefix), + rendered.ends_with(&test_case.expected_suffix), + ), + (true, true), + "{}: {rendered}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_resolving_outcome_then_exit_code_matches() { + let test_cases = [ + OutcomeTestCase { + description: "identical rows pass", + models: vec![model(Some(changed_rows(0)), &[])], + query_row_comparison: false, + expected_outcome: ("pass", 0), + }, + OutcomeTestCase { + description: "unequal rows are findings", + models: vec![model(Some(changed_rows(1)), &[])], + query_row_comparison: false, + expected_outcome: ("findings", 1), + }, + OutcomeTestCase { + description: "query schema change without row comparison is incomplete", + models: vec![model(None, &["extra"])], + query_row_comparison: true, + expected_outcome: ("incomplete", 2), + }, + OutcomeTestCase { + description: "model schema change is a finding", + models: vec![model(None, &["extra"])], + query_row_comparison: false, + expected_outcome: ("findings", 1), + }, + ]; + + for test_case in test_cases { + let (outcome, code) = diff_exit_code(&test_case.models, test_case.query_row_comparison); + + assert_eq!( + (outcome.as_str(), code), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_changed_rows_when_rendering_summary_then_layout_matches_rich() { + let test_cases = [SummaryTestCase { + description: "one unequal row", + model: model(Some(changed_rows(1)), &[]), + expected_lines: &[ + " Model orders ", + "┏━┯━┯━┯━┯━┓╌╌╌┏━┯━┯━┯━┯━┓╭──┬──────────┬─────────╮╌╮ ", + "┃ │ │ │ │ ┃ ≠ ┃ │ │ │ │ ┃│ 2│ equal │ (66.67%)│ │ ", + "│ amount │ mismatches=1 │ match=66.67% │", + ], + }]; + + for test_case in test_cases { + let text: String = render_diff_text( + &[test_case.model], + &document(), + &DiffTextOptions { + mode_label: "full".to_owned(), + max_column_examples: 3, + max_row_only_examples: 3, + ..DiffTextOptions::default() + }, + ); + + let rendered: Vec<&str> = text.lines().collect(); + let found: Vec<&str> = test_case + .expected_lines + .iter() + .copied() + .filter(|line| rendered.contains(line)) + .collect(); + + assert_eq!( + found, + test_case.expected_lines.to_vec(), + "{}: {text}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_rendering_json_then_documents_are_sorted_and_safe() { + let test_cases = [ + JsonTestCase { + description: "non-finite cursor bound becomes a tagged object", + document: render_diff_json(&[model(Some(cursor_rows()), &[])], &document()), + expected_fragment: "\"start\": {\n \"kind\": \"non_finite_number\",\n \"value\": \"NaN\"\n }", + }, + JsonTestCase { + description: "error document keeps the incomplete outcome", + document: render_diff_error_json(&DiffErrorFacts { + status: "incomplete".to_owned(), + code: "C254".to_owned(), + message: "key is not unique".to_owned(), + help: None, + size_guard: None, + }), + expected_fragment: "{\n \"error\": {\n \"code\": \"C254\",\n \"message\": \"key is not unique\"\n },\n \"models\": [],\n \"outcome\": \"incomplete\",\n \"query_comparisons\": [],\n \"schema_version\": 1,\n \"status\": \"incomplete\"\n}", + }, + ]; + + for test_case in test_cases { + let text: String = test_case.document.unwrap_or_default(); + + assert!( + text.contains(test_case.expected_fragment), + "{}: {text}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/tests/test_types.rs b/crates/sqlbuild-cli/src/diff_report/tests/test_types.rs new file mode 100644 index 0000000000..7fab5a725f --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/tests/test_types.rs @@ -0,0 +1,44 @@ +use sqlbuild_core::json::errors::JsonEmitError; + +use crate::diff_report::models::{DiffExampleOptions, DiffModelFacts}; + +/// Two unequal values, how they render, and what the rendering reports. +pub(crate) struct EvidenceTestCase { + pub(crate) description: &'static str, + pub(crate) left: String, + pub(crate) right: String, + pub(crate) options: DiffExampleOptions, + pub(crate) expected_left: String, + pub(crate) expected_right: String, + pub(crate) expected_first_difference: Option, +} + +/// Diff results and the outcome and exit code they produce. +pub(crate) struct OutcomeTestCase { + pub(crate) description: &'static str, + pub(crate) models: Vec, + pub(crate) query_row_comparison: bool, + pub(crate) expected_outcome: (&'static str, i32), +} + +/// One model and fragments its terminal summary must contain. +pub(crate) struct SummaryTestCase { + pub(crate) description: &'static str, + pub(crate) model: DiffModelFacts, + pub(crate) expected_lines: &'static [&'static str], +} + +/// A key value and how its rendering starts and ends. +pub(crate) struct KeyTestCase { + pub(crate) description: &'static str, + pub(crate) key: String, + pub(crate) expected_prefix: String, + pub(crate) expected_suffix: String, +} + +/// A rendered JSON document and a fragment it must contain. +pub(crate) struct JsonTestCase { + pub(crate) description: &'static str, + pub(crate) document: Result, + pub(crate) expected_fragment: &'static str, +} diff --git a/crates/sqlbuild-cli/src/diff_report/types.rs b/crates/sqlbuild-cli/src/diff_report/types.rs new file mode 100644 index 0000000000..c39f179e83 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/types.rs @@ -0,0 +1,9 @@ +//! Tuple shapes of diff facts. + +use crate::diff_report::models::DiffValue; + +/// One named value, such as a key column of a sampled row. +pub type DiffNamedValue = (String, DiffValue); + +/// A schema column whose type differs: name, `from` type and `to` type. +pub type DiffTypeChange = (String, String, String); diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 01a09fdfc7..4bbb72fffb 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -4,6 +4,7 @@ pub mod argv; pub mod compute_logs; +pub mod diff_report; pub mod dispatch; pub mod launcher; pub mod replay; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index eed0b86fe3..c947ab597f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -32,7 +32,6 @@ const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const CODED_ERROR_MODULE: &str = "sqlbuild.presentation.main.coded_error_text"; const TRANSIENT_LINE_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; const SKILLS_MODULE: &str = "sqlbuild._native"; -const DIFF_JSON_MODULE: &str = "sqlbuild.cli.commands._helpers.diff.json_output"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; @@ -195,7 +194,7 @@ fn render_diff_error<'py>( fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { call_with( - &imported(py, DIFF_JSON_MODULE, "render_diff_error_json")?, + &imported(py, NATIVE_MODULE, "render_diff_error_json")?, fields, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index c2c3ce805a..b560bc368b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -4,8 +4,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; use crate::bindings::_helpers::{ - check, clone, command_runtime, cost_report, dbt_interop, debug_command, freshness, janitor, - load, presentation_text, result_output, workspace_commands, + check, clone, command_runtime, cost_report, dbt_interop, debug_command, diff_report, freshness, + janitor, load, presentation_text, result_output, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -22,6 +22,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { cost_report::registration::register(module)?; debug_command::registration::register(module)?; dbt_interop::registration::register(module)?; + diff_report::registration::register(module)?; command_runtime::registration::register(module)?; presentation_text::registration::register(module)?; workspace_commands::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs new file mode 100644 index 0000000000..56bd8aeea2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs @@ -0,0 +1,273 @@ +//! Python diff results read as plain diff facts. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, + PyStringMethods, PyTuple, +}; +use sqlbuild_cli::diff_report::models::{ + DiffChangedCell, DiffColumnFacts, DiffCoverageFacts, DiffModelFacts, DiffRowFacts, + DiffSampleRow, DiffValue, +}; +use sqlbuild_cli::diff_report::types::DiffNamedValue; +use sqlbuild_core::json::main::python_float_repr::python_float_repr; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +fn text_of(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.str()?.to_string_lossy().into_owned()) +} + +fn strings(value: &Bound<'_, PyAny>) -> PyResult> { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(text_of(&item?)?); + } + Ok(items) +} + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + text_of(&value).map(Some) +} + +/// A JSON object key as `json.dumps` writes it. +fn json_key(key: &Bound<'_, PyAny>) -> PyResult { + if let Ok(flag) = key.downcast::() { + return Ok(if flag.is_true() { "true" } else { "false" }.to_owned()); + } + if key.is_none() { + return Ok("null".to_owned()); + } + if key.is_instance_of::() { + return Ok(python_float_repr(key.extract()?)); + } + text_of(key) +} + +/// One value as `json.dumps(default=...)` encodes it: dates as ISO text, other objects as `str()`. +pub(crate) fn json_of(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(JsonValue::Null); + } + if let Ok(text) = value.downcast::() { + return Ok(JsonValue::String(text.to_string_lossy().into_owned())); + } + if let Ok(flag) = value.downcast::() { + return Ok(JsonValue::Bool(flag.is_true())); + } + if value.is_instance_of::() { + let decimal: String = value + .call_method0("__index__")? + .str()? + .to_string_lossy() + .into_owned(); + if let Some(integer) = JsonInteger::parse(&decimal) { + return Ok(JsonValue::Integer(integer)); + } + } + if value.is_instance_of::() { + return Ok(JsonValue::Float(value.extract()?)); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(json_of(&item?)?); + } + return Ok(JsonValue::Array(items)); + } + if let Ok(mapping) = value.downcast::() { + let mut entries: Vec<(String, JsonValue)> = Vec::new(); + for (key, item) in mapping.iter() { + entries.push((json_key(&key)?, json_of(&item)?)); + } + return Ok(JsonValue::Object(entries)); + } + let py: Python<'_> = value.py(); + if value.is_instance(&imported(py, "datetime", "date")?)? { + return Ok(JsonValue::String(text_of( + &value.call_method0("isoformat")?, + )?)); + } + Ok(JsonValue::String(text_of(value)?)) +} + +fn diff_value(value: &Bound<'_, PyAny>) -> PyResult { + Ok(DiffValue { + text: text_of(value)?, + json: json_of(value)?, + }) +} + +fn optional_value(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + diff_value(&value).map(Some) +} + +fn named_values(value: &Bound<'_, PyAny>) -> PyResult> { + let mut pairs: Vec = Vec::new(); + for pair in value.try_iter()? { + let (name, item): (Bound<'_, PyAny>, Bound<'_, PyAny>) = pair?.extract()?; + pairs.push((text_of(&name)?, diff_value(&item)?)); + } + Ok(pairs) +} + +fn key_samples(value: &Bound<'_, PyAny>) -> PyResult>> { + let mut samples: Vec> = Vec::new(); + for sample in value.try_iter()? { + samples.push(named_values(&sample?)?); + } + Ok(samples) +} + +fn coverage(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(DiffCoverageFacts { + row_count: value.getattr("row_count")?.extract()?, + minimum_cursor: optional_value(value.getattr("minimum_cursor")?)?, + maximum_cursor: optional_value(value.getattr("maximum_cursor")?)?, + })) +} + +fn column(value: &Bound<'_, PyAny>) -> PyResult { + let tolerance: Bound<'_, PyAny> = value.getattr("tolerance")?; + Ok(DiffColumnFacts { + name: text_of(&value.getattr("name")?)?, + mismatched_count: value.getattr("mismatched_count")?.extract()?, + tolerance: if tolerance.is_none() { + None + } else { + Some(( + optional_text(tolerance.getattr("absolute")?)?, + optional_text(tolerance.getattr("relative")?)?, + )) + }, + }) +} + +fn coverage_differs(rows: &Bound<'_, PyAny>) -> PyResult { + let left: Bound<'_, PyAny> = rows.getattr("left_coverage")?; + let right: Bound<'_, PyAny> = rows.getattr("right_coverage")?; + if left.is_none() || right.is_none() { + return Ok(false); + } + Ok(left + .getattr("minimum_cursor")? + .ne(right.getattr("minimum_cursor")?)? + || left + .getattr("maximum_cursor")? + .ne(right.getattr("maximum_cursor")?)?) +} + +fn row_facts(rows: Bound<'_, PyAny>) -> PyResult> { + if rows.is_none() { + return Ok(None); + } + let count = |name: &str| -> PyResult { rows.getattr(name)?.extract() }; + let mut columns: Vec = Vec::new(); + for item in rows.getattr("column_results")?.try_iter()? { + columns.push(column(&item?)?); + } + let sampling: Bound<'_, PyAny> = rows.getattr("sampling")?; + Ok(Some(DiffRowFacts { + left_count: count("left_count")?, + right_count: count("right_count")?, + joined_count: count("joined_count")?, + equal_count: count("equal_count")?, + unequal_count: count("unequal_count")?, + left_only_count: count("left_only_count")?, + right_only_count: count("right_only_count")?, + columns, + population_count: count("population_count")?, + compared_count: count("compared_count")?, + sampling: if sampling.is_none() { + None + } else { + Some(( + sampling.getattr("row_limit")?.extract()?, + sampling.getattr("seed")?.extract()?, + )) + }, + left_coverage: coverage(rows.getattr("left_coverage")?)?, + right_coverage: coverage(rows.getattr("right_coverage")?)?, + cursor_column: optional_text(rows.getattr("cursor_column")?)?, + cursor_start: optional_value(rows.getattr("cursor_start")?)?, + cursor_end: optional_value(rows.getattr("cursor_end")?)?, + exhaustive: rows.getattr("is_exhaustive")?.is_truthy()?, + coverage_differs: coverage_differs(&rows)?, + })) +} + +fn sample_row(value: &Bound<'_, PyAny>) -> PyResult { + let mut changed_cells: Vec = Vec::new(); + for cell in value.getattr("changed_cells")?.try_iter()? { + let cell: Bound<'_, PyAny> = cell?; + changed_cells.push(DiffChangedCell { + name: text_of(&cell.getattr("name")?)?, + left: diff_value(&cell.getattr("left_value")?)?, + right: diff_value(&cell.getattr("right_value")?)?, + }); + } + Ok(DiffSampleRow { + key_values: named_values(&value.getattr("key_values")?)?, + changed_cells, + }) +} + +fn model_facts(model: &Bound<'_, PyAny>) -> PyResult { + let schema: Bound<'_, PyAny> = model.getattr("schema_result")?; + let names = |attribute: &str| -> PyResult> { + let mut names: Vec = Vec::new(); + for column in schema.getattr(attribute)?.try_iter()? { + names.push(text_of(&column?.getattr("name")?)?); + } + Ok(names) + }; + let mut type_changes: Vec<(String, String, String)> = Vec::new(); + for pair in schema.getattr("type_changed_columns")?.try_iter()? { + let (left, right): (Bound<'_, PyAny>, Bound<'_, PyAny>) = pair?.extract()?; + type_changes.push(( + text_of(&left.getattr("name")?)?, + text_of(&left.getattr("type")?)?, + text_of(&right.getattr("type")?)?, + )); + } + let mut unequal_samples: Vec = Vec::new(); + for sample in model.getattr("unequal_row_samples")?.try_iter()? { + unequal_samples.push(sample_row(&sample?)?); + } + Ok(DiffModelFacts { + name: text_of(&model.getattr("name")?)?, + left_relation: text_of(&model.getattr("left_relation")?)?, + right_relation: text_of(&model.getattr("right_relation")?)?, + added_columns: names("added_columns")?, + removed_columns: names("removed_columns")?, + type_changes, + unique_key: strings(&model.getattr("unique_key")?)?, + rows: row_facts(model.getattr("row_result")?)?, + unequal_samples, + left_only_keys: key_samples(&model.getattr("left_only_key_samples")?)?, + right_only_keys: key_samples(&model.getattr("right_only_key_samples")?)?, + bounded_fallback: model.getattr("bounded_fallback")?.is_truthy()?, + excluded_columns: strings(&model.getattr("excluded_columns")?)?, + input_kind: text_of(&model.getattr("input_kind")?)?, + unkeyed: model.getattr("unkeyed")?.is_truthy()?, + }) +} + +/// Every model result of a `DiffExecutionResult`. +pub(crate) fn diff_facts(result: &Bound<'_, PyAny>) -> PyResult> { + let mut models: Vec = Vec::new(); + for model in result.getattr("model_results")?.try_iter()? { + models.push(model_facts(&model?)?); + } + Ok(models) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs new file mode 100644 index 0000000000..35d9ab8112 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod facts; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs new file mode 100644 index 0000000000..406899bf13 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs @@ -0,0 +1,265 @@ +//! `sqb diff` output: the terminal summary, JSON documents, output publication and exit codes. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, + wrap_pyfunction, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_cli::diff_report::main::diff_exit_code::diff_exit_code; +use sqlbuild_cli::diff_report::main::render_diff_error_json::render_diff_error_json as error_document; +use sqlbuild_cli::diff_report::main::render_diff_json::render_diff_json; +use sqlbuild_cli::diff_report::main::render_diff_text::render_diff_text; +use sqlbuild_cli::diff_report::models::{ + DiffDocumentOptions, DiffErrorFacts, DiffExampleOptions, DiffModelFacts, DiffTextOptions, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::diff_report::facts::{diff_facts, json_of}; +use crate::bindings::_helpers::frontier::encoder::emit_error; + +const DEFAULT_MAX_VALUE_LENGTH: usize = 160; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const TEXT_OPTIONS: (&[&str], &[&str]) = ( + &[ + "result", + "from_label", + "to_label", + "mode_label", + "use_color", + "verbose", + "max_column_examples", + "max_row_only_examples", + ], + &["max_value_length", "suppress_example_values", "outcome"], +); +const JSON_OPTIONS: (&[&str], &[&str]) = ( + &["result", "from_label", "to_label"], + &[ + "max_value_length", + "suppress_example_values", + "phase_seconds", + "outcome", + ], +); +const ERROR_OPTIONS: (&[&str], &[&str]) = + (&["status", "code", "message"], &["help_text", "size_guard"]); +const WRITE_OPTIONS: (&[&str], &[&str]) = ( + &[ + "request", + "from_label", + "to_label", + "max_column_examples", + "max_row_only_examples", + "result", + ], + &["phase_seconds", "outcome"], +); +const EXIT_OPTIONS: (&[&str], &[&str]) = (&["result"], &["query_row_comparison"]); + +fn optional_string(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +fn phase_seconds(value: Bound<'_, PyAny>) -> PyResult>> { + if value.is_none() { + return Ok(None); + } + let mapping: Bound<'_, PyDict> = value.downcast_into()?; + let mut phases: Vec<(String, f64)> = Vec::new(); + for (phase, seconds) in mapping.iter() { + phases.push((phase.extract()?, seconds.extract()?)); + } + Ok(Some(phases)) +} + +/// The example options of explicit `max_value_length`/`suppress_example_values` arguments. +fn example_options(arguments: &KeywordArguments<'_>) -> PyResult { + Ok(DiffExampleOptions { + max_value_length: if arguments.passed("max_value_length")? { + { + let limit: Bound<'_, PyAny> = arguments.value("max_value_length")?; + if limit.is_none() { + None + } else { + Some(limit.extract()?) + } + } + } else { + Some(DEFAULT_MAX_VALUE_LENGTH) + }, + suppress_values: arguments.flag("suppress_example_values")?, + }) +} + +fn document_options( + arguments: &KeywordArguments<'_>, + examples: DiffExampleOptions, +) -> PyResult { + Ok(DiffDocumentOptions { + from_label: arguments.value("from_label")?.extract()?, + to_label: arguments.value("to_label")?.extract()?, + examples, + outcome: optional_string(arguments.value("outcome")?)?, + phase_seconds: phase_seconds(arguments.value("phase_seconds")?)?, + }) +} + +fn json_text(models: &[DiffModelFacts], options: &DiffDocumentOptions) -> PyResult { + render_diff_json(models, options).map_err(emit_error) +} + +/// Render the concise terminal summary of a diff result. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_diff_output(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "render_diff_output", options, TEXT_OPTIONS)?; + let models: Vec = diff_facts(&arguments.value("result")?)?; + let document: DiffDocumentOptions = document_options(&arguments, example_options(&arguments)?)?; + Ok(render_diff_text( + &models, + &document, + &DiffTextOptions { + mode_label: arguments.value("mode_label")?.extract()?, + use_color: arguments.flag("use_color")?, + verbose: arguments.flag("verbose")?, + max_column_examples: arguments.value("max_column_examples")?.extract()?, + max_row_only_examples: arguments.value("max_row_only_examples")?.extract()?, + }, + )) +} + +/// Render one stable structured diff document. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_diff_json_output(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "render_diff_json_output", options, JSON_OPTIONS)?; + let models: Vec = diff_facts(&arguments.value("result")?)?; + json_text( + &models, + &document_options(&arguments, example_options(&arguments)?)?, + ) +} + +/// Render a machine-readable incomplete or execution-failure outcome. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_diff_error_json(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "render_diff_error_json", options, ERROR_OPTIONS)?; + let size_guard: Bound<'_, PyAny> = arguments.value("size_guard")?; + error_document(&DiffErrorFacts { + status: arguments.value("status")?.extract()?, + code: arguments.value("code")?.extract()?, + message: arguments.value("message")?.extract()?, + help: optional_string(arguments.value("help_text")?)?, + size_guard: if size_guard.is_none() { + None + } else { + Some(json_of(&size_guard)?) + }, + }) + .map_err(emit_error) +} + +fn mode_label(request: &Bound<'_, PyAny>) -> PyResult { + if request.getattr("schema_only")?.is_truthy()? { + return Ok("schema-only".to_owned()); + } + let bounded: Bound<'_, PyAny> = request.getattr("bounded")?; + if bounded.is_truthy()? { + return Ok(format!("bounded {}", bounded.str()?)); + } + Ok("full".to_owned()) +} + +fn write_stdout(py: Python<'_>, text: &str) -> PyResult<()> { + imported(py, "sys", "stdout")?.call_method1("write", (format!("{text}\n"),))?; + Ok(()) +} + +/// Write the requested JSON file, then print the JSON document or the terminal summary. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_diff_output(py: Python<'_>, options: Option>) -> PyResult<()> { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "write_diff_output", options, WRITE_OPTIONS)?; + let request: Bound<'_, PyAny> = arguments.value("request")?; + let max_value_length: Bound<'_, PyAny> = request.getattr("max_value_length")?; + let examples: DiffExampleOptions = DiffExampleOptions { + max_value_length: if request.getattr("full_example_values")?.is_truthy()? { + None + } else if max_value_length.is_truthy()? { + Some(max_value_length.extract()?) + } else { + Some(DEFAULT_MAX_VALUE_LENGTH) + }, + suppress_values: request.getattr("suppress_example_values")?.is_truthy()?, + }; + let document: DiffDocumentOptions = document_options(&arguments, examples)?; + let models: Vec = diff_facts(&arguments.value("result")?)?; + let path: Bound<'_, PyAny> = request.getattr("json_output_path")?; + if !path.is_none() { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("parents", true)?; + keywords.set_item("exist_ok", true)?; + path.getattr("parent")? + .call_method("mkdir", (), Some(&keywords))?; + let encoding: Bound<'_, PyDict> = PyDict::new(py); + encoding.set_item("encoding", "utf-8")?; + path.call_method( + "write_text", + (format!("{}\n", json_text(&models, &document)?),), + Some(&encoding), + )?; + } + if request.getattr("json_output")?.is_truthy()? { + return write_stdout(py, &json_text(&models, &document)?); + } + let use_color: bool = !request.getattr("no_color")?.is_truthy()? + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?; + write_stdout( + py, + &render_diff_text( + &models, + &document, + &DiffTextOptions { + mode_label: mode_label(&request)?, + use_color, + verbose: request.getattr("verbose")?.is_truthy()?, + max_column_examples: arguments.value("max_column_examples")?.extract()?, + max_row_only_examples: arguments.value("max_row_only_examples")?.extract()?, + }, + ), + ) +} + +/// `(outcome, exit_code)` of a finished diff, with query row-comparison semantics on request. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_diff_outcome( + py: Python<'_>, + options: Option>, +) -> PyResult<(String, i32)> { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "resolve_diff_outcome", options, EXIT_OPTIONS)?; + let models: Vec = diff_facts(&arguments.value("result")?)?; + Ok(diff_exit_code( + &models, + arguments.flag("query_row_comparison")?, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_diff_output, module)?)?; + module.add_function(wrap_pyfunction!(render_diff_json_output, module)?)?; + module.add_function(wrap_pyfunction!(render_diff_error_json, module)?)?; + module.add_function(wrap_pyfunction!(write_diff_output, module)?)?; + module.add_function(wrap_pyfunction!(resolve_diff_outcome, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index e26581ccea..27686c92dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod dag_artifact; pub(crate) mod dbt_interop; pub(crate) mod debug_command; pub(crate) mod diagnostics; +pub(crate) mod diff_report; pub(crate) mod discovery; pub(crate) mod fingerprints; pub(crate) mod freshness; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3d0c05ab19..15ffe53341 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -21,6 +21,7 @@ from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.models import ( CheckCommandRequest, CloneCommandRequest, + DiffCommandRequest, FreshnessCommandRequest, JanitorCommandRequest, LineageCommandRequest, @@ -222,6 +223,7 @@ from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult +from sqlbuild.executor.diff.models import DiffExecutionResult from sqlbuild.executor.janitor.models import JanitorExecutionResult, JanitorPlan from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes.models import ( @@ -1516,6 +1518,52 @@ class NativeProgressProjector: ) -> float | None: ... def close(self) -> None: ... +def render_diff_output( + *, + result: DiffExecutionResult, + from_label: str, + to_label: str, + mode_label: str, + use_color: bool, + verbose: bool, + max_column_examples: int, + max_row_only_examples: int, + max_value_length: int | None = ..., + suppress_example_values: bool = ..., + outcome: str | None = ..., +) -> str: ... +def render_diff_json_output( + *, + result: DiffExecutionResult, + from_label: str, + to_label: str, + max_value_length: int | None = ..., + suppress_example_values: bool = ..., + phase_seconds: dict[str, float] | None = ..., + outcome: str | None = ..., +) -> str: ... +def render_diff_error_json( + *, + status: str, + code: str, + message: str, + help_text: str | None = ..., + size_guard: dict[str, object] | None = ..., +) -> str: ... +def write_diff_output( + *, + request: DiffCommandRequest, + from_label: str, + to_label: str, + max_column_examples: int, + max_row_only_examples: int, + result: DiffExecutionResult, + phase_seconds: dict[str, float] | None = ..., + outcome: str | None = ..., +) -> None: ... +def resolve_diff_outcome( + *, result: DiffExecutionResult, query_row_comparison: bool = ... +) -> tuple[str, int]: ... def machine_output_mode(args: CliNamespace, /) -> bool: ... def current_native_progress_projector() -> NativeProgressProjector | None: ... diff --git a/src/sqlbuild/cli/commands/_helpers/diff/evidence.py b/src/sqlbuild/cli/commands/_helpers/diff/evidence.py deleted file mode 100644 index 0ae5378b32..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/evidence.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Bounded, diff-aware rendering for row-diff evidence.""" - -from __future__ import annotations - -import hashlib - -from sqlbuild.cli.commands.models import DiffExampleRenderOptions, RenderedDiffExampleValue - -_MAX_RENDERED_KEY_LENGTH: int = 256 - - -def render_example_pair( - *, - left_value: object, - right_value: object, - options: DiffExampleRenderOptions, -) -> tuple[RenderedDiffExampleValue, RenderedDiffExampleValue]: - """Render two unequal values around their first differing character.""" - - left_text: str = str(left_value) - right_text: str = str(right_value) - first_difference: int | None = _first_difference(left=left_text, right=right_text) - return ( - _render_value(text=left_text, first_difference=first_difference, options=options), - _render_value(text=right_text, first_difference=first_difference, options=options), - ) - - -def display_example_value(value: RenderedDiffExampleValue) -> str: - """Render one bounded value for terminal output.""" - - if value.suppressed: - return f"" - rendered: str = value.text or "" - if value.truncated: - return f"{rendered} [truncated; {value.original_length:,} chars]" - return rendered - - -def render_key_value(value: object) -> str: - """Keep ordinary keys complete and identify exceptionally long shortened keys.""" - - text: str = str(value) - if len(text) <= _MAX_RENDERED_KEY_LENGTH: - return text - digest: str = hashlib.sha256(text.encode("utf-8")).hexdigest()[:12] - return f"{text[:_MAX_RENDERED_KEY_LENGTH]}… [truncated; {len(text):,} chars; sha256:{digest}]" - - -def example_value_payload(value: RenderedDiffExampleValue) -> dict[str, object]: - """Build stable structured evidence for one rendered value.""" - - return { - "value": value.text, - "original_length": value.original_length, - "truncated": value.truncated, - "suppressed": value.suppressed, - "first_difference": value.first_difference, - } - - -def _render_value( - *, - text: str, - first_difference: int | None, - options: DiffExampleRenderOptions, -) -> RenderedDiffExampleValue: - original_length: int = len(text) - if options.suppress_values: - return RenderedDiffExampleValue( - text=None, - original_length=original_length, - truncated=False, - suppressed=True, - first_difference=first_difference, - ) - limit: int | None = options.max_value_length - if limit is None or original_length <= limit: - return RenderedDiffExampleValue( - text=text, - original_length=original_length, - truncated=False, - suppressed=False, - first_difference=first_difference, - ) - difference: int = first_difference or 0 - context_before: int = min(limit // 3, max(limit - 1, 0)) - start: int = min(max(difference - context_before, 0), original_length - limit) - end: int = start + limit - fragment: str = text[start:end] - rendered: str = f"{'…' if start else ''}{fragment}{'…' if end < original_length else ''}" - return RenderedDiffExampleValue( - text=rendered, - original_length=original_length, - truncated=True, - suppressed=False, - first_difference=first_difference, - ) - - -def _first_difference(*, left: str, right: str) -> int | None: - index: int - for index, (left_char, right_char) in enumerate(zip(left, right, strict=False)): - if left_char != right_char: - return index - if len(left) != len(right): - return min(len(left), len(right)) - return None diff --git a/src/sqlbuild/cli/commands/_helpers/diff/execution.py b/src/sqlbuild/cli/commands/_helpers/diff/execution.py index 81400a38dc..6e0381fa28 100644 --- a/src/sqlbuild/cli/commands/_helpers/diff/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/diff/execution.py @@ -9,6 +9,7 @@ from typing import Any from sqlbuild._native import ( + resolve_diff_outcome, resolve_project_connection_config, resolve_target_connection_config, run_diff_pipeline, @@ -402,7 +403,9 @@ def execute_query_diff( "query diff comparison completed without a result", code="C256" ) result: DiffExecutionResult = DiffExecutionResult(model_results=(model_result,)) - outcome, exit_code = _resolve_query_diff_outcome(request=request, result=result) + outcome, exit_code = resolve_diff_outcome( + result=result, query_row_comparison=not request.schema_only + ) return QueryDiffRunOutcome( result=result, phase_seconds=phase_seconds, @@ -468,45 +471,6 @@ def _cleanup_query_diff_artifacts( return errors, elapsed -def _resolve_query_diff_outcome( - *, request: DiffCommandRequest, result: DiffExecutionResult -) -> tuple[str, int]: - if not request.schema_only and _query_schema_prevented_row_comparison(result=result): - return "incomplete", 2 - if _query_result_has_findings(result=result): - return "findings", 1 - return "pass", 0 - - -def _query_schema_prevented_row_comparison(*, result: DiffExecutionResult) -> bool: - return any( - model.row_result is None - and ( - model.schema_result.added_columns - or model.schema_result.removed_columns - or model.schema_result.type_changed_columns - ) - for model in result.model_results - ) - - -def _query_result_has_findings(*, result: DiffExecutionResult) -> bool: - return any( - model.schema_result.added_columns - or model.schema_result.removed_columns - or model.schema_result.type_changed_columns - or ( - model.row_result is not None - and ( - model.row_result.unequal_count - or model.row_result.left_only_count - or model.row_result.right_only_count - ) - ) - for model in result.model_results - ) - - def _inspect_query_diff_inputs( *, adapter: BaseAdapter, diff --git a/src/sqlbuild/cli/commands/_helpers/diff/json_output.py b/src/sqlbuild/cli/commands/_helpers/diff/json_output.py deleted file mode 100644 index c3c872d1bd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/json_output.py +++ /dev/null @@ -1,314 +0,0 @@ -"""Structured JSON output for diff results.""" - -from __future__ import annotations - -import json -import math -from datetime import date, datetime -from pathlib import Path - -from sqlbuild.adapter.contract.models import RowDiffCoverage, RowDiffResult, RowDiffSampleRow -from sqlbuild.cli.commands._helpers.diff.evidence import ( - example_value_payload, - render_example_pair, -) -from sqlbuild.cli.commands._helpers.diff.output import has_diff_failures -from sqlbuild.cli.commands.constants import DEFAULT_DIFF_MAX_VALUE_LENGTH -from sqlbuild.cli.commands.models import DiffExampleRenderOptions -from sqlbuild.executor.diff.constants import DIFF_INPUT_KIND_QUERY -from sqlbuild.executor.diff.models import DiffExecutionResult, ModelDiffResult - -_DEFAULT_EXAMPLE_RENDER_OPTIONS: DiffExampleRenderOptions = DiffExampleRenderOptions( - max_value_length=DEFAULT_DIFF_MAX_VALUE_LENGTH -) - - -def render_diff_json_output( - *, - result: DiffExecutionResult, - from_label: str, - to_label: str, - example_render_options: DiffExampleRenderOptions = _DEFAULT_EXAMPLE_RENDER_OPTIONS, - phase_seconds: dict[str, float] | None = None, - outcome: str | None = None, -) -> str: - """Render one stable structured diff document.""" - - return json.dumps( - _json_safe_value( - _diff_payload( - result=result, - from_label=from_label, - to_label=to_label, - example_render_options=example_render_options, - phase_seconds=phase_seconds, - outcome=outcome, - ) - ), - allow_nan=False, - indent=2, - sort_keys=True, - default=_json_default, - ) - - -def render_diff_error_json( - *, - status: str, - code: str, - message: str, - help_text: str | None = None, - size_guard: dict[str, object] | None = None, -) -> str: - """Render a machine-readable incomplete or execution-failure outcome.""" - - error: dict[str, object] = {"code": code, "message": message} - if help_text is not None: - error["help"] = help_text - payload: dict[str, object] = { - "schema_version": 1, - "status": status, - "outcome": status, - "error": error, - "models": [], - "query_comparisons": [], - } - if size_guard is not None: - payload["size_guard"] = size_guard - return json.dumps(payload, indent=2, sort_keys=True) - - -def _json_safe_value(value: object) -> object: - if isinstance(value, float) and not math.isfinite(value): - if math.isnan(value): - label: str = "NaN" - elif value > 0: - label = "Infinity" - else: - label = "-Infinity" - return {"kind": "non_finite_number", "value": label} - if isinstance(value, dict): - return {key: _json_safe_value(item) for key, item in value.items()} - if isinstance(value, list | tuple): - return [_json_safe_value(item) for item in value] - return value - - -def write_diff_json_output( - *, - path: Path | None, - result: DiffExecutionResult, - from_label: str, - to_label: str, - example_render_options: DiffExampleRenderOptions = _DEFAULT_EXAMPLE_RENDER_OPTIONS, - phase_seconds: dict[str, float] | None = None, - outcome: str | None = None, -) -> None: - """Write a stable structured diff document when a path is requested.""" - - if path is None: - return - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - render_diff_json_output( - result=result, - from_label=from_label, - to_label=to_label, - example_render_options=example_render_options, - phase_seconds=phase_seconds, - outcome=outcome, - ) - + "\n", - encoding="utf-8", - ) - - -def _diff_payload( - *, - result: DiffExecutionResult, - from_label: str, - to_label: str, - example_render_options: DiffExampleRenderOptions, - phase_seconds: dict[str, float] | None, - outcome: str | None, -) -> dict[str, object]: - model_payloads: list[dict[str, object]] = [ - _model_payload( - model=model, - from_label=from_label, - to_label=to_label, - example_render_options=example_render_options, - ) - for model in result.model_results - ] - resolved_outcome: str = outcome or ("findings" if has_diff_failures(result) else "pass") - status: str = { - "pass": "no_differences_found", - "findings": "differences_found", - }.get(resolved_outcome, resolved_outcome) - payload: dict[str, object] = { - "schema_version": 1, - "status": status, - "outcome": resolved_outcome, - "from": from_label, - "to": to_label, - "models": model_payloads, - } - query_payloads: list[dict[str, object]] = [ - model_payload - for model, model_payload in zip(result.model_results, model_payloads, strict=True) - if model.input_kind == DIFF_INPUT_KIND_QUERY - ] - if query_payloads: - payload["input_kind"] = DIFF_INPUT_KIND_QUERY - payload["query_comparisons"] = query_payloads - if phase_seconds is not None: - payload["phase_seconds"] = { - phase: round(seconds, 6) for phase, seconds in sorted(phase_seconds.items()) - } - return payload - - -def _model_payload( - *, - model: ModelDiffResult, - from_label: str, - to_label: str, - example_render_options: DiffExampleRenderOptions, -) -> dict[str, object]: - rows: RowDiffResult | None = model.row_result - payload: dict[str, object] = { - "input_kind": model.input_kind, - "name": model.name, - "from_label": from_label, - "to_label": to_label, - "from_relation": model.left_relation, - "to_relation": model.right_relation, - "unique_key": list(model.unique_key), - "unkeyed": model.unkeyed, - "excluded_columns": list(model.excluded_columns), - "schema": { - "added_columns": [column.name for column in model.schema_result.added_columns], - "removed_columns": [column.name for column in model.schema_result.removed_columns], - "type_changed_columns": [ - {"name": left.name, "from_type": left.type, "to_type": right.type} - for left, right in model.schema_result.type_changed_columns - ], - }, - } - if rows is None: - payload["comparison_scope"] = "schema_only" - return payload - payload["comparison_scope"] = "exhaustive" if rows.is_exhaustive else "sampled" - payload["coverage"] = { - "requested": { - "cursor_column": rows.cursor_column, - "start": rows.cursor_start, - "end": rows.cursor_end, - }, - "from": _coverage_payload(coverage=rows.left_coverage), - "to": _coverage_payload(coverage=rows.right_coverage), - "differs": _coverage_differs(rows=rows), - } - payload["sampling"] = { - "configured_row_limit": rows.sampling.row_limit if rows.sampling is not None else None, - "seed": rows.sampling.seed if rows.sampling is not None else None, - "population_keys": None if model.unkeyed else rows.population_count, - "compared_keys": None if model.unkeyed else rows.compared_count, - "coverage_ratio": ( - rows.compared_count / rows.population_count if rows.population_count else 1.0 - ), - "exhaustive": rows.is_exhaustive, - } - payload["rows"] = { - "from_count": rows.left_count, - "to_count": rows.right_count, - "joined_count": rows.equal_count + rows.unequal_count, - "equal_count": rows.equal_count, - "unequal_count": rows.unequal_count, - "from_only_count": rows.left_only_count, - "to_only_count": rows.right_only_count, - "changed_columns": {column.name: column.mismatched_count for column in rows.column_results}, - "column_mismatches": [ - {"column": column.name, "rows": column.mismatched_count} - for column in rows.column_results - if column.mismatched_count > 0 - ], - } - payload["examples"] = _examples_payload( - model=model, - options=example_render_options, - ) - return payload - - -def _examples_payload( - *, model: ModelDiffResult, options: DiffExampleRenderOptions -) -> dict[str, object]: - unequal_rows: list[dict[str, object]] = [] - values_truncated: bool = False - sample_row: RowDiffSampleRow - for sample_row in model.unequal_row_samples: - cells: list[dict[str, object]] = [] - for cell in sample_row.changed_cells: - left_value, right_value = render_example_pair( - left_value=cell.left_value, - right_value=cell.right_value, - options=options, - ) - values_truncated = values_truncated or left_value.truncated or right_value.truncated - cells.append( - { - "column": cell.name, - "from": example_value_payload(left_value), - "to": example_value_payload(right_value), - } - ) - unequal_rows.append( - { - "key": {name: value for name, value in sample_row.key_values}, - "changed_cells": cells, - } - ) - rows: RowDiffResult | None = model.row_result - return { - "unequal_rows": unequal_rows, - "from_only_keys": [dict(sample) for sample in model.left_only_key_samples], - "to_only_keys": [dict(sample) for sample in model.right_only_key_samples], - "truncated_by_count": bool( - rows is not None - and ( - rows.unequal_count > len(model.unequal_row_samples) - or rows.left_only_count > len(model.left_only_key_samples) - or rows.right_only_count > len(model.right_only_key_samples) - ) - ), - "values_truncated": values_truncated, - "values_suppressed": options.suppress_values, - "max_value_length": options.max_value_length, - } - - -def _coverage_payload(*, coverage: RowDiffCoverage | None) -> dict[str, object] | None: - if coverage is None: - return None - return { - "row_count": coverage.row_count, - "minimum_cursor": coverage.minimum_cursor, - "maximum_cursor": coverage.maximum_cursor, - } - - -def _coverage_differs(*, rows: RowDiffResult) -> bool: - if rows.cursor_column is None or rows.left_coverage is None or rows.right_coverage is None: - return False - return ( - rows.left_coverage.minimum_cursor != rows.right_coverage.minimum_cursor - or rows.left_coverage.maximum_cursor != rows.right_coverage.maximum_cursor - ) - - -def _json_default(value: object) -> str: - if isinstance(value, datetime | date): - return value.isoformat() - return str(value) diff --git a/src/sqlbuild/cli/commands/_helpers/diff/output.py b/src/sqlbuild/cli/commands/_helpers/diff/output.py deleted file mode 100644 index b235670339..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/output.py +++ /dev/null @@ -1,652 +0,0 @@ -"""Output helpers for the diff command.""" - -from __future__ import annotations - -from rich import box -from rich.columns import Columns as RichColumns -from rich.console import Console, Group, RenderableType -from rich.panel import Panel -from rich.table import Table -from rich.text import Text - -from sqlbuild.adapter.contract.models import ( - RowDiffColumnResult, - RowDiffCoverage, - RowDiffResult, - RowDiffSampleCell, - RowDiffSampleRow, - SchemaDiffResult, -) -from sqlbuild.cli.commands._helpers.diff.evidence import ( - display_example_value, - render_example_pair, - render_key_value, -) -from sqlbuild.cli.commands.constants import DEFAULT_DIFF_MAX_VALUE_LENGTH -from sqlbuild.cli.commands.models import DiffExampleRenderOptions -from sqlbuild.executor.diff.constants import DIFF_INPUT_KIND_QUERY -from sqlbuild.executor.diff.models import DiffExecutionResult, ModelDiffResult - -_WIDTH: int = 110 -_TOP_CHANGED_COLUMNS: int = 5 -_RICH_TITLE_STYLE: str = "bold" -_RICH_OBJECT_STYLE: str = "bold cyan" -_RICH_SECTION_STYLE: str = "bold" -_RICH_ADDED_STYLE: str = "green" -_RICH_REMOVED_STYLE: str = "red" -_RICH_MUTED_STYLE: str = "dim" -_DEFAULT_EXAMPLE_RENDER_OPTIONS: DiffExampleRenderOptions = DiffExampleRenderOptions( - max_value_length=DEFAULT_DIFF_MAX_VALUE_LENGTH -) - - -def render_diff_output( - *, - result: DiffExecutionResult, - from_label: str, - to_label: str, - mode_label: str, - use_color: bool, - verbose: bool, - max_column_examples: int, - max_row_only_examples: int, - example_render_options: DiffExampleRenderOptions = _DEFAULT_EXAMPLE_RENDER_OPTIONS, - outcome: str | None = None, -) -> str: - """Render a concise Rich terminal summary for a diff result.""" - - console: Console = Console(force_terminal=use_color, color_system="auto", width=_WIDTH) - with console.capture() as capture: - if not result.model_results: - console.print("No models selected for diff.") - else: - console.print( - Panel( - Text("SQLBuild Diff Summary", style=_RICH_TITLE_STYLE, justify="center"), - box=box.HEAVY, - ) - ) - console.print( - Text(f"{from_label} vs {to_label}", style=_RICH_OBJECT_STYLE, justify="center") - ) - input_label: str = ( - "query pairs" - if all(item.input_kind == DIFF_INPUT_KIND_QUERY for item in result.model_results) - else "models" - ) - console.print( - Text(f"selected {input_label}: {len(result.model_results):,}", justify="center") - ) - resolved_outcome: str = outcome or ("findings" if has_diff_failures(result) else "pass") - console.print(Text(f"outcome: {resolved_outcome}", justify="center")) - console.print() - model_result: ModelDiffResult - for index, model_result in enumerate(result.model_results): - if index > 0: - console.print() - console.print(Text("─" * _WIDTH, style=_RICH_MUTED_STYLE)) - console.print() - _render_model_result( - console=console, - model_result=model_result, - from_label=from_label, - to_label=to_label, - mode_label=mode_label, - verbose=verbose, - max_column_examples=max_column_examples, - max_row_only_examples=max_row_only_examples, - example_render_options=example_render_options, - ) - return capture.get().rstrip() - - -def has_diff_failures(result: DiffExecutionResult) -> bool: - """Return true when any selected model has schema or row differences.""" - - model_result: ModelDiffResult - for model_result in result.model_results: - schema_result: SchemaDiffResult = model_result.schema_result - if ( - schema_result.added_columns - or schema_result.removed_columns - or schema_result.type_changed_columns - ): - return True - row_result: RowDiffResult | None = model_result.row_result - if row_result is not None and ( - row_result.unequal_count or row_result.left_only_count or row_result.right_only_count - ): - return True - return False - - -def _render_model_result( - *, - console: Console, - model_result: ModelDiffResult, - from_label: str, - to_label: str, - mode_label: str, - verbose: bool, - max_column_examples: int, - max_row_only_examples: int, - example_render_options: DiffExampleRenderOptions, -) -> None: - console.print( - _render_overview( - model_result=model_result, - mode_label=mode_label, - ) - ) - console.print() - _print_section(console=console, title="Schemas", content=_render_schema_summary(model_result)) - if model_result.row_result is not None: - _print_section( - console=console, - title="Coverage", - content=_render_coverage( - rows=model_result.row_result, - from_label=from_label, - to_label=to_label, - ), - ) - _print_section( - console=console, - title="Rows" if model_result.row_result.is_exhaustive else "Sampled Rows", - content=_render_rows( - rows=model_result.row_result, - from_label=from_label, - to_label=to_label, - ), - ) - _print_section( - console=console, - title="Changed Columns", - content=_render_changed_columns( - model_result=model_result, - verbose=verbose, - max_column_examples=max_column_examples, - example_render_options=example_render_options, - ), - ) - if verbose and model_result.unequal_row_samples: - example_content: RenderableType | None = _render_examples( - model_result=model_result, - max_column_examples=max_column_examples, - example_render_options=example_render_options, - ) - if example_content is not None: - _print_section( - console=console, - title="Examples", - content=example_content, - ) - if model_result.left_only_key_samples: - _print_section( - console=console, - title=f"{from_label} only", - content=_render_side_only_samples( - side_label=from_label, - key_samples=model_result.left_only_key_samples, - max_row_only_examples=max_row_only_examples, - ), - ) - if model_result.right_only_key_samples: - _print_section( - console=console, - title=f"{to_label} only", - content=_render_side_only_samples( - side_label=to_label, - key_samples=model_result.right_only_key_samples, - max_row_only_examples=max_row_only_examples, - ), - ) - - -def _render_overview(*, model_result: ModelDiffResult, mode_label: str) -> RenderableType: - table: Table = Table.grid(padding=(0, 2)) - table.add_column(style=_RICH_OBJECT_STYLE, justify="right") - table.add_column() - table.add_row( - "Input" if model_result.input_kind == DIFF_INPUT_KIND_QUERY else "Model", - model_result.name, - ) - table.add_row("Key", _primary_key_label(model_result)) - table.add_row("Comparison", _mode_display(model_result=model_result, mode_label=mode_label)) - if model_result.bounded_fallback: - table.add_row("Fallback", "no cursor configured; used full row diff") - if model_result.excluded_columns: - table.add_row("Excluded", ", ".join(model_result.excluded_columns)) - if model_result.row_result is not None: - table.add_row( - "Scope", - _row_scope_label(rows=model_result.row_result, unkeyed=model_result.unkeyed), - ) - tolerance_label: str | None = _tolerance_label(model_result.row_result) - if tolerance_label is not None: - table.add_row("Tolerances", tolerance_label) - return table - - -def _primary_key_label(model_result: ModelDiffResult) -> str: - if not model_result.unique_key: - return "" - return ", ".join(model_result.unique_key) - - -def _mode_display(*, model_result: ModelDiffResult, mode_label: str) -> str: - if model_result.bounded_fallback: - return f"{mode_label} (fallback to full row diff)" - return mode_label - - -def _tolerance_label(row_result: RowDiffResult | None) -> str | None: - if row_result is None: - return None - labels: list[str] = [] - column: RowDiffColumnResult - for column in row_result.column_results: - if column.tolerance is None: - continue - parts: list[str] = [] - if column.tolerance.absolute is not None: - parts.append(f"absolute={column.tolerance.absolute}") - if column.tolerance.relative is not None: - parts.append(f"relative={column.tolerance.relative}") - labels.append(f"{column.name} {' '.join(parts)}") - if not labels: - return None - return ", ".join(labels) - - -def _row_scope_label(*, rows: RowDiffResult, unkeyed: bool = False) -> str: - if unkeyed: - return f"exhaustive ({rows.population_count:,} multiset rows)" - if rows.sampling is None: - return f"exhaustive ({rows.population_count:,} union keys)" - if rows.is_exhaustive: - return ( - f"exhaustive ({rows.compared_count:,} union keys; " - f"configured sample limit {rows.sampling.row_limit:,})" - ) - percentage: str = _format_percentage( - numerator=rows.compared_count, - denominator=rows.population_count, - ) - return ( - f"sampled {rows.compared_count:,} of {rows.population_count:,} union keys " - f"({percentage}; seed {rows.sampling.seed})" - ) - - -def _render_coverage(*, rows: RowDiffResult, from_label: str, to_label: str) -> RenderableType: - if rows.left_coverage is None or rows.right_coverage is None: - return Text("Coverage was not collected.", style="italic") - table: Table = Table.grid(padding=(0, 2)) - table.add_column(style=_RICH_OBJECT_STYLE) - table.add_column(justify="right") - table.add_column() - table.add_row("Side", "Rows", "Cursor extent") - table.add_row( - from_label, - f"{rows.left_coverage.row_count:,}", - _cursor_extent(rows=rows, coverage=rows.left_coverage), - ) - table.add_row( - to_label, - f"{rows.right_coverage.row_count:,}", - _cursor_extent(rows=rows, coverage=rows.right_coverage), - ) - renderables: list[RenderableType] = [table] - if _cursor_coverage_differs(rows=rows): - renderables.extend( - ( - "", - Text( - "Warning: cursor coverage differs between sides; comparison continued " - "without narrowing the requested bounds.", - style="yellow", - ), - ) - ) - return Group(*renderables) - - -def _cursor_extent(*, rows: RowDiffResult, coverage: RowDiffCoverage) -> str: - if rows.cursor_column is None: - return "" - minimum: str = ( - str(coverage.minimum_cursor) if coverage.minimum_cursor is not None else "" - ) - maximum: str = ( - str(coverage.maximum_cursor) if coverage.maximum_cursor is not None else "" - ) - requested: str = _requested_cursor_bounds(rows=rows) - return f"{minimum} to {maximum} (requested {requested})" - - -def _requested_cursor_bounds(*, rows: RowDiffResult) -> str: - start: str = str(rows.cursor_start) if rows.cursor_start is not None else "unbounded" - end: str = str(rows.cursor_end) if rows.cursor_end is not None else "unbounded" - return f"{start} to {end}" - - -def _cursor_coverage_differs(*, rows: RowDiffResult) -> bool: - if rows.cursor_column is None or rows.left_coverage is None or rows.right_coverage is None: - return False - return ( - rows.left_coverage.minimum_cursor != rows.right_coverage.minimum_cursor - or rows.left_coverage.maximum_cursor != rows.right_coverage.maximum_cursor - ) - - -def _render_schema_summary(model_result: ModelDiffResult) -> RenderableType: - schema_result: SchemaDiffResult = model_result.schema_result - schema_diff_count: int = ( - len(schema_result.added_columns) - + len(schema_result.removed_columns) - + len(schema_result.type_changed_columns) - ) - if ( - not schema_result.added_columns - and not schema_result.removed_columns - and not schema_result.type_changed_columns - ): - return Text("No schema differences.", style="italic") - lines: list[str] = [f"schema differences: {schema_diff_count:,}"] - if schema_result.added_columns: - lines.append(f"added columns: {len(schema_result.added_columns):,}") - if schema_result.removed_columns: - lines.append(f"removed columns: {len(schema_result.removed_columns):,}") - if schema_result.type_changed_columns: - lines.append(f"type changes: {len(schema_result.type_changed_columns):,}") - return Group(*lines) - - -def _print_section(*, console: Console, title: str, content: RenderableType) -> None: - console.print(Text(title, style=_RICH_SECTION_STYLE)) - console.print(Text("▔" * len(title), style=_RICH_SECTION_STYLE)) - console.print(content) - console.print() - - -def _render_rows(*, rows: RowDiffResult, from_label: str, to_label: str) -> RenderableType: - counts: Table = Table.grid(padding=(0, 2)) - counts.add_column(justify="center") - counts.add_column(justify="center") - counts.add_column(justify="center") - counts.add_row(f"{from_label} count", "", f"{to_label} count") - counts.add_row( - f"{rows.left_count:,}", - _render_delta_label(left_count=rows.left_count, right_count=rows.right_count), - f"{rows.right_count:,}", - ) - - joined_count: int = rows.equal_count + rows.unequal_count - renderables: list[RenderableType] = [] - left_box: Table = Table(show_header=False, padding=0, box=box.HEAVY_EDGE) - right_box: Table = Table(show_header=False, padding=0, box=box.HEAVY_EDGE) - for _ in range(5): - left_box.add_column() - right_box.add_column() - - if rows.left_only_count > 0: - left_box.add_row(*([Text("-", style=_RICH_REMOVED_STYLE)] * 5)) - left_box.add_section() - if rows.equal_count > 0: - left_box.add_row(*([" "] * 5)) - left_box.add_section() - right_box.add_row(*([" "] * 5)) - right_box.add_section() - if rows.unequal_count > 0: - left_box.add_row(*([" "] * 5)) - right_box.add_row(*([" "] * 5)) - right_box.add_section() - if rows.right_only_count > 0: - right_box.add_row(*([Text("+", style=_RICH_ADDED_STYLE)] * 5)) - - renderables.append(left_box) - renderables.append(_render_row_separator(rows)) - renderables.append(Group("\n", right_box) if rows.left_only_count > 0 else right_box) - renderables.append( - _render_row_stats( - rows=rows, joined_count=joined_count, from_label=from_label, to_label=to_label - ) - ) - renderables.append(_render_joined_annotation(rows=rows, joined_count=joined_count)) - - return Group(counts, "", RichColumns(renderables, padding=0), "", f"joined: {joined_count:,}") - - -def _render_delta_label(*, left_count: int, right_count: int) -> str: - if left_count == 0: - return "(no baseline)" - if left_count == right_count: - return "(no change)" - if right_count > left_count: - return f"(+{((right_count / left_count) - 1):.2%})" - return f"(-{(1 - (right_count / left_count)):.2%})" - - -def _render_row_separator(rows: RowDiffResult) -> RenderableType: - joined: list[RenderableType] = [] - if rows.left_only_count > 0: - joined.append("\n") - if rows.equal_count > 0: - joined.append("╌" * 3) - joined.append(Text(" = ", style=_RICH_SECTION_STYLE)) - if rows.unequal_count > 0: - joined.append("╌" * 3) - joined.append(Text(" ≠ ", style=_RICH_SECTION_STYLE)) - joined.append("╌" * 3) - return Group(*joined) - - -def _render_row_stats( - *, - rows: RowDiffResult, - joined_count: int, - from_label: str, - to_label: str, -) -> RenderableType: - table: Table = Table(show_header=False, box=box.ROUNDED, padding=(0, 0, 0, 1)) - table.add_column(justify="right") - table.add_column(justify="left") - table.add_column(justify="right") - table.add_row( - f"{rows.left_only_count:,}", - f"{from_label} only", - f"({_format_percentage(numerator=rows.left_only_count, denominator=rows.left_count)})", - ) - table.add_section() - table.add_row( - f"{rows.equal_count:,}", - "equal", - f"({_format_percentage(numerator=rows.equal_count, denominator=joined_count)})", - ) - table.add_section() - table.add_row( - f"{rows.unequal_count:,}", - "unequal", - f"({_format_percentage(numerator=rows.unequal_count, denominator=joined_count)})", - ) - table.add_section() - table.add_row( - f"{rows.right_only_count:,}", - f"{to_label} only", - f"({_format_percentage(numerator=rows.right_only_count, denominator=rows.right_count)})", - ) - return table - - -def _render_joined_annotation(*, rows: RowDiffResult, joined_count: int) -> RenderableType: - if joined_count == 0: - return "" - lines: list[RenderableType] = [] - if rows.left_only_count > 0: - lines.append("\n") - lines.append("╌╮") - lines.append(" │") - if rows.equal_count > 0 and rows.unequal_count > 0: - lines.append(f"╌├╴ {joined_count:,} joined") - lines.append(" │") - lines.append("╌╯") - return Group(*lines) - - -def _render_changed_columns( - *, - model_result: ModelDiffResult, - verbose: bool, - max_column_examples: int, - example_render_options: DiffExampleRenderOptions, -) -> RenderableType: - row_result: RowDiffResult | None = model_result.row_result - if row_result is None: - return Text("Schema-only diff; row comparison skipped.", style="italic") - mismatched_columns: tuple[RowDiffColumnResult, ...] = tuple( - sorted( - (column for column in row_result.column_results if column.mismatched_count > 0), - key=lambda column: (-column.mismatched_count, column.name), - ) - ) - if not mismatched_columns: - message: str = ( - "No changed columns." - if row_result.is_exhaustive - else "No changed columns in sampled keys." - ) - return Text(message, style="italic") - table: Table = Table(show_header=False) - table.add_column(style=_RICH_OBJECT_STYLE, max_width=32, overflow="fold") - table.add_column(justify="right") - table.add_column(justify="right") - visible_columns: tuple[RowDiffColumnResult, ...] = mismatched_columns[:_TOP_CHANGED_COLUMNS] - column: RowDiffColumnResult - for column in visible_columns: - match_rate: str = _format_percentage( - numerator=_matching_rows(row_result=row_result, column=column), - denominator=row_result.joined_count, - ) - table.add_row( - column.name, - f"mismatches={column.mismatched_count:,}", - f"match={match_rate}", - ) - if len(mismatched_columns) > len(visible_columns): - table.add_section() - table.add_row( - Text("...", style=_RICH_MUTED_STYLE), - Text( - f"and {len(mismatched_columns) - len(visible_columns):,} more", - style=_RICH_MUTED_STYLE, - ), - Text("", style=_RICH_MUTED_STYLE), - ) - if not verbose: - example_content: RenderableType | None = _render_examples( - model_result=model_result, - max_column_examples=max_column_examples, - visible_column_names=tuple(column.name for column in visible_columns), - example_render_options=example_render_options, - ) - if example_content is not None: - return Group( - table, - "", - example_content, - "", - Text("Use --verbose to show more example row changes.", style=_RICH_MUTED_STYLE), - ) - return table - return table - - -def _matching_rows(*, row_result: RowDiffResult, column: RowDiffColumnResult) -> int: - return max(row_result.joined_count - column.mismatched_count, 0) - - -def _format_percentage(*, numerator: int | float, denominator: int | float) -> str: - if denominator == 0: - return "0.00%" - return f"{(numerator / denominator):.2%}" - - -def _render_examples( - *, - model_result: ModelDiffResult, - max_column_examples: int, - visible_column_names: tuple[str, ...] | None = None, - example_render_options: DiffExampleRenderOptions, -) -> RenderableType | None: - blocks: list[RenderableType] = [] - grouped_examples: dict[str, list[str]] = {} - sample_row: RowDiffSampleRow - for sample_row in model_result.unequal_row_samples: - key_label: str = ", ".join( - f"{key}={render_key_value(value)}" for key, value in sample_row.key_values - ) - cell: RowDiffSampleCell - for cell in sample_row.changed_cells: - left_value, right_value = render_example_pair( - left_value=cell.left_value, - right_value=cell.right_value, - options=example_render_options, - ) - difference_label: str = ( - f" (first difference at character {left_value.first_difference + 1:,})" - if (left_value.truncated or right_value.truncated) - and left_value.first_difference is not None - else "" - ) - grouped_examples.setdefault(cell.name, []).append( - f"{key_label} | {display_example_value(left_value)} -> " - f"{display_example_value(right_value)}{difference_label}" - ) - column_name: str - for column_name in sorted(grouped_examples): - if visible_column_names is not None and column_name not in visible_column_names: - continue - blocks.append(Text(column_name, style=_RICH_OBJECT_STYLE)) - all_examples: list[str] = grouped_examples[column_name] - visible_examples: list[str] = all_examples[:max_column_examples] - example: str - for example in visible_examples: - blocks.append(Text(f" - {example}")) - if len(all_examples) > len(visible_examples): - blocks.append( - Text( - f" showing {len(visible_examples):,} of {len(all_examples):,} examples", - style=_RICH_MUTED_STYLE, - ) - ) - blocks.append("") - return Group(*blocks[:-1]) if blocks else None - - -def _render_side_only_samples( - *, - side_label: str, - key_samples: tuple[tuple[tuple[str, object], ...], ...], - max_row_only_examples: int, -) -> RenderableType: - blocks: list[RenderableType] = [] - visible_samples: tuple[tuple[tuple[str, object], ...], ...] = key_samples[ - :max_row_only_examples - ] - sample: tuple[tuple[str, object], ...] - for sample in visible_samples: - blocks.append( - Text(" - " + " | ".join(f"{key}={render_key_value(value)}" for key, value in sample)) - ) - if len(key_samples) > len(visible_samples): - truncation_message: str = ( - f" showing {len(visible_samples):,} of {len(key_samples):,} {side_label} only rows" - ) - blocks.append( - Text( - truncation_message, - style=_RICH_MUTED_STYLE, - ) - ) - return Group(*blocks) if blocks else Text("No side-only samples collected.", style="italic") diff --git a/src/sqlbuild/cli/commands/_helpers/diff/outputs.py b/src/sqlbuild/cli/commands/_helpers/diff/outputs.py deleted file mode 100644 index 05e8c73ccd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/outputs.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Diff command output writing and exit-code phases.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.diff.json_output import ( - render_diff_json_output, - write_diff_json_output, -) -from sqlbuild.cli.commands._helpers.diff.output import has_diff_failures, render_diff_output -from sqlbuild.cli.commands.constants import DEFAULT_DIFF_MAX_VALUE_LENGTH -from sqlbuild.cli.commands.models import ( - DiffCommandRequest, - DiffExampleRenderOptions, - DirectDiffPreparation, -) -from sqlbuild.executor.diff.models import DiffExecutionResult -from sqlbuild.presentation.main.supports_color import supports_color - - -def write_direct_diff_output( - *, - request: DiffCommandRequest, - preparation: DirectDiffPreparation, - result: DiffExecutionResult, - phase_seconds: dict[str, float] | None = None, - outcome: str | None = None, -) -> None: - """Write direct diff output.""" - - example_render_options: DiffExampleRenderOptions = _example_render_options(request=request) - write_diff_json_output( - path=request.json_output_path, - result=result, - from_label=preparation.from_target, - to_label=preparation.to_target, - example_render_options=example_render_options, - phase_seconds=phase_seconds, - outcome=outcome, - ) - if request.json_output: - print( - render_diff_json_output( - result=result, - from_label=preparation.from_target, - to_label=preparation.to_target, - example_render_options=example_render_options, - phase_seconds=phase_seconds, - outcome=outcome, - ) - ) - else: - print( - render_diff_output( - result=result, - from_label=preparation.from_target, - to_label=preparation.to_target, - mode_label=_mode_label(request=request), - use_color=not request.no_color and supports_color(), - verbose=request.verbose, - max_column_examples=preparation.effective_max_column_examples, - max_row_only_examples=preparation.effective_max_row_only_examples, - example_render_options=example_render_options, - outcome=outcome, - ) - ) - - -def resolve_diff_exit_code(result: DiffExecutionResult) -> int: - """Resolve the diff exit code from failure state.""" - - return 1 if has_diff_failures(result) else 0 - - -def _example_render_options(*, request: DiffCommandRequest) -> DiffExampleRenderOptions: - max_value_length: int | None = ( - None - if request.full_example_values - else request.max_value_length or DEFAULT_DIFF_MAX_VALUE_LENGTH - ) - return DiffExampleRenderOptions( - max_value_length=max_value_length, - suppress_values=request.suppress_example_values, - ) - - -def _mode_label(*, request: DiffCommandRequest) -> str: - if request.schema_only: - return "schema-only" - if request.bounded: - return f"bounded {request.bounded}" - return "full" diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index 22b5406d2c..ac9c101da2 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -68,7 +68,6 @@ READ_ONLY_QUERY_ROOT_KEYS: frozenset[str] = frozenset( {"except", "intersect", "select", "union", "values"} ) -DEFAULT_DIFF_MAX_VALUE_LENGTH: int = 160 QUERY_DIFF_INCOMPLETE_PREPARATION_CODES: frozenset[str] = frozenset( {"C229", "C230", "C231", "C232", "C233", "C234", "C235", "C240", "C244"} ) diff --git a/src/sqlbuild/cli/commands/main/inspection/_diff.py b/src/sqlbuild/cli/commands/main/inspection/_diff.py index e48ffabe89..be19e22119 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_diff.py +++ b/src/sqlbuild/cli/commands/main/inspection/_diff.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import resolve_diff_outcome, write_diff_output from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.cli.commands._helpers.diff.execution import ( execute_direct_diff, @@ -13,10 +14,6 @@ is_query_diff_request, resolve_diff_invocation, ) -from sqlbuild.cli.commands._helpers.diff.outputs import ( - resolve_diff_exit_code, - write_direct_diff_output, -) from sqlbuild.cli.commands.constants import ( QUERY_DIFF_INCOMPLETE_EXECUTION_CODES, QUERY_DIFF_INCOMPLETE_PREPARATION_CODES, @@ -99,23 +96,12 @@ def run_diff(request: DiffCommandRequest) -> int: except Exception as error: raise QueryDiffExecutionError(str(error), code="C255") from error try: - write_direct_diff_output( + write_diff_output( request=request, - preparation=DirectDiffPreparation( - from_target=query_preparation.left_label, - to_target=query_preparation.right_label, - adapter=query_preparation.adapter, - left_project=None, - right_project=None, - selected_names=( - f"{query_preparation.left_label} vs {query_preparation.right_label}", - ), - connection_config=query_preparation.connection_config, - effective_max_column_examples=query_preparation.effective_max_column_examples, - effective_max_row_only_examples=( - query_preparation.effective_max_row_only_examples - ), - ), + from_label=query_preparation.left_label, + to_label=query_preparation.right_label, + max_column_examples=query_preparation.effective_max_column_examples, + max_row_only_examples=query_preparation.effective_max_row_only_examples, result=query_outcome.result, phase_seconds=query_outcome.phase_seconds, outcome=query_outcome.outcome, @@ -134,12 +120,17 @@ def run_diff(request: DiffCommandRequest) -> int: request=request, preparation=direct_preparation, ) - write_direct_diff_output( + write_diff_output( request=request, - preparation=direct_preparation, + from_label=direct_preparation.from_target, + to_label=direct_preparation.to_target, + max_column_examples=direct_preparation.effective_max_column_examples, + max_row_only_examples=direct_preparation.effective_max_row_only_examples, result=result, ) - return resolve_diff_exit_code(result) + exit_code: int + _outcome, exit_code = resolve_diff_outcome(result=result) + return exit_code def _is_query_key_preflight_error(*, error: AdapterUserError) -> bool: diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index e03115a344..ff1b64dfdd 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -453,25 +453,6 @@ class QueryDiffPreparation: effective_max_row_only_examples: int -@dataclass(frozen=True) -class DiffExampleRenderOptions: - """Bounded example rendering controls independent of comparison execution.""" - - max_value_length: int | None - suppress_values: bool = False - - -@dataclass(frozen=True) -class RenderedDiffExampleValue: - """One safely bounded value plus explicit information-loss metadata.""" - - text: str | None - original_length: int - truncated: bool - suppressed: bool - first_difference: int | None - - @dataclass(frozen=True) class QueryDiffRunOutcome: """Raw-query result plus measured lifecycle phase durations.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/diff/helpers.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/diff/helpers.py index e2081558d3..02c910a1fc 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/diff/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/diff/helpers.py @@ -4,6 +4,7 @@ from decimal import Decimal from typing import cast +from sqlbuild._native import render_diff_output from sqlbuild.adapter.contract.models import ( ColumnInfo, RowDiffColumnResult, @@ -13,7 +14,6 @@ RowDiffTolerance, SchemaDiffResult, ) -from sqlbuild.cli.commands._helpers.diff.output import render_diff_output from sqlbuild.executor.diff.models import DiffExecutionResult, ModelDiffResult from tests.integration.src.sqlbuild.cli.commands._helpers.diff._test_types import ( DiffOutputIntegrationTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py index 51d0a04e06..676b13d528 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Literal -from sqlbuild.cli.commands.models import DiffExampleRenderOptions from sqlbuild.executor.diff.models import DiffExecutionResult, FullDiffSizeLimits @@ -19,19 +18,6 @@ class RenderDiffOutputTestCase: expected_fragments: tuple[str, ...] -@dataclass(frozen=True) -class RenderDiffEvidenceTestCase: - description: str - left_value: object - right_value: object - options: DiffExampleRenderOptions - expected_left_fragment: str - expected_right_fragment: str - expected_first_difference: int | None - expected_truncated: bool - expected_suppressed: bool - - @dataclass(frozen=True) class FullDiffSizeLimitsTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_evidence.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_evidence.py deleted file mode 100644 index c4de2b5e67..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_evidence.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Tests for bounded diff example evidence.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.diff.evidence import ( - display_example_value, - render_example_pair, -) -from sqlbuild.cli.commands.models import DiffExampleRenderOptions, RenderedDiffExampleValue -from tests.unit.src.sqlbuild.cli.commands._helpers.diff._test_types import ( - RenderDiffEvidenceTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderDiffEvidenceTestCase( - description="late text difference keeps nearby context", - left_value=("a" * 200) + "left-tail", - right_value=("a" * 200) + "right-tail", - options=DiffExampleRenderOptions(max_value_length=30), - expected_left_fragment="left-tail", - expected_right_fragment="right-tail", - expected_first_difference=200, - expected_truncated=True, - expected_suppressed=False, - ), - RenderDiffEvidenceTestCase( - description="suppression retains lengths without values", - left_value="left-private-value", - right_value="right-private-value", - options=DiffExampleRenderOptions(max_value_length=30, suppress_values=True), - expected_left_fragment="18 chars", - expected_right_fragment="19 chars", - expected_first_difference=0, - expected_truncated=False, - expected_suppressed=True, - ), - RenderDiffEvidenceTestCase( - description="explicit unlimited rendering preserves complete values", - left_value=("a" * 200) + "left-tail", - right_value=("a" * 200) + "right-tail", - options=DiffExampleRenderOptions(max_value_length=None), - expected_left_fragment=("a" * 200) + "left-tail", - expected_right_fragment=("a" * 200) + "right-tail", - expected_first_difference=200, - expected_truncated=False, - expected_suppressed=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_unequal_values_when_rendering_evidence_then_bounds_are_explicit( - test_case: RenderDiffEvidenceTestCase, -) -> None: - left: RenderedDiffExampleValue - right: RenderedDiffExampleValue - left, right = render_example_pair( - left_value=test_case.left_value, - right_value=test_case.right_value, - options=test_case.options, - ) - - assert test_case.expected_left_fragment in display_example_value(left) - assert test_case.expected_right_fragment in display_example_value(right) - assert left.first_difference == test_case.expected_first_difference - assert right.first_difference == test_case.expected_first_difference - assert left.truncated is test_case.expected_truncated - assert right.truncated is test_case.expected_truncated - assert left.suppressed is test_case.expected_suppressed - assert right.suppressed is test_case.expected_suppressed diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_output.py index f071a0b79f..bc344f1ca8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_output.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import render_diff_output from sqlbuild.adapter.contract.models import ( ColumnInfo, RowDiffColumnResult, @@ -13,7 +14,6 @@ RowDiffTolerance, SchemaDiffResult, ) -from sqlbuild.cli.commands._helpers.diff.output import render_diff_output from sqlbuild.executor.diff.models import DiffExecutionResult, ModelDiffResult from tests.unit.src.sqlbuild.cli.commands._helpers.diff._test_types import ( RenderDiffOutputTestCase, From 1d3c2b7d0110a4bcbce83bc6f510f50a69d87b6a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:52:54 +0100 Subject: [PATCH 22/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index d23870b8e4..0eb4259082 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 84185 +remaining_lines = 82955 [[stays]] path = "executor" From b409ea6cf16b95d249272b35717645bd0cb161d5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:12:59 +0100 Subject: [PATCH 23/37] refactor: import janitor, load, clone and freshness Python by complete module paths --- .../_helpers/clone/clone_execution.rs | 33 ++++++--------- .../_helpers/freshness/freshness_observe.rs | 37 +++++++++-------- .../_helpers/janitor/janitor_command.rs | 41 +++++++++---------- .../bindings/_helpers/load/load_command.rs | 21 ++++++---- 4 files changed, 66 insertions(+), 66 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 3bc319110d..742ff72c25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -18,7 +18,11 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; -const CLONE_PACKAGE: &str = "sqlbuild.executor.clone.main"; +const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main.build_retention_requests"; +const TRANSIENT_MODELS_MODULE: &str = + "sqlbuild.executor.clone.main.build_destination_transient_models"; +const FINGERPRINTING_MODULE: &str = "sqlbuild.executor.clone.main.fingerprinting"; +const EXECUTE_MODULE: &str = "sqlbuild.executor.clone.main.execute"; const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; @@ -171,13 +175,6 @@ fn target_config<'py>(invocation: &Invocation<'py>) -> PyResult( - py: Python<'py>, - (module, name): (&str, &str), -) -> PyResult> { - imported(py, &format!("{CLONE_PACKAGE}.{module}"), name) -} - /// The executor input for the whole clone, streaming items through `on_item`. fn execution_input<'py>( invocation: &Invocation<'py>, @@ -194,12 +191,10 @@ fn execution_input<'py>( .map(|entry| entry?.getattr("name")) .collect::>()?; let retention: Bound<'py, PyAny> = call_with( - &clone_function( + &imported( py, - ( - "build_retention_requests", - "build_destination_retention_requests", - ), + RETENTION_REQUESTS_MODULE, + "build_destination_retention_requests", )?, &[ ("project", project.clone()), @@ -231,12 +226,10 @@ fn execution_input<'py>( ], )?; let transient: Bound<'py, PyAny> = call_with( - &clone_function( + &imported( py, - ( - "build_destination_transient_models", - "build_destination_transient_models", - ), + TRANSIENT_MODELS_MODULE, + "build_destination_transient_models", )?, &[("project", project.clone())], )?; @@ -327,7 +320,7 @@ fn copy_fingerprints<'py>( )?; let copied: PyResult> = within(&lifecycle, |_| { call_with( - &clone_function(py, ("fingerprinting", "copy_clone_fingerprints"))?, + &imported(py, FINGERPRINTING_MODULE, "copy_clone_fingerprints")?, &fields, ) }); @@ -370,7 +363,7 @@ pub(crate) fn execute_clone_plan<'py>( ), )?; call_with( - &clone_function(py, ("execute", "execute_clone"))?, + &imported(py, EXECUTE_MODULE, "execute_clone")?, &[("inputs", input)], ) })(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs index c4d55866a3..99f698dc69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs @@ -16,8 +16,9 @@ use sqlbuild_core::freshness_text::models::{ FreshnessSourceResult, FreshnessStatus, StateComparison, }; -const FRESHNESS_PACKAGE: &str = "sqlbuild.compiler.source_freshness.main"; -const NATIVE: &str = "_native"; +const ADAPTER_OBSERVATION_MODULE: &str = + "sqlbuild.compiler.source_freshness.main.adapter_observation"; +const OBSERVATION_MODULE: &str = "sqlbuild.compiler.source_freshness.main.observation"; const NATIVE_MODULE: &str = "sqlbuild._native"; const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; @@ -46,11 +47,7 @@ fn freshness_function<'py>( py: Python<'py>, (module, name): (&str, &str), ) -> PyResult> { - if module == NATIVE { - return py.import(NATIVE_MODULE)?.getattr(name); - } - py.import(format!("{FRESHNESS_PACKAGE}.{module}"))? - .getattr(name) + py.import(module)?.getattr(name) } fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { @@ -136,16 +133,18 @@ fn record<'py>( let normalize: Bound<'py, PyDict> = PyDict::new(py); normalize.set_item("value", observation.getattr("data_version")?)?; normalize.set_item("value_kind", observation.getattr("value_kind")?)?; - let data_version: Bound<'py, PyAny> = - freshness_function(py, (NATIVE, "normalize_source_freshness_data_version"))? - .call((), Some(&normalize))?; + let data_version: Bound<'py, PyAny> = freshness_function( + py, + (NATIVE_MODULE, "normalize_source_freshness_data_version"), + )? + .call((), Some(&normalize))?; let hash: Bound<'py, PyDict> = PyDict::new(py); hash.set_item("source_name", observation.getattr("source_name")?)?; hash.set_item("strategy", observation.getattr("strategy")?)?; hash.set_item("value_kind", observation.getattr("value_kind")?)?; hash.set_item("data_version", &data_version)?; let data_version_hash: Bound<'py, PyAny> = - freshness_function(py, (NATIVE, "source_freshness_data_version_hash"))? + freshness_function(py, (NATIVE_MODULE, "source_freshness_data_version_hash"))? .call((), Some(&hash))?; let fields: Bound<'py, PyDict> = PyDict::new(py); fields.set_item("source_name", observation.getattr("source_name")?)?; @@ -201,7 +200,7 @@ fn observed_result( age.set_item("data_version", observation.getattr("data_version")?)?; age.set_item("observed_at", observation.getattr("observed_at")?)?; let age_status: Bound<'_, PyAny> = - freshness_function(py, (NATIVE, "evaluate_source_freshness_age_policy"))? + freshness_function(py, (NATIVE_MODULE, "evaluate_source_freshness_age_policy"))? .call((), Some(&age))?; let previous: Option> = previous_record(state, ¤t)?; let comparison: StateComparison = match &previous { @@ -213,7 +212,7 @@ fn observed_result( options.set_item("current_record", ¤t)?; options.set_item("lag_tolerance", &lag_tolerance)?; let equivalent: bool = - freshness_function(py, (NATIVE, "source_freshness_records_equivalent"))? + freshness_function(py, (NATIVE_MODULE, "source_freshness_records_equivalent"))? .call((), Some(&options))? .is_truthy()?; if equivalent { @@ -270,7 +269,10 @@ fn adapter_batch<'py>( options.set_item("observed_at", &observer.observed_at)?; match freshness_function( py, - ("adapter_observation", "observe_adapter_sources_freshness"), + ( + ADAPTER_OBSERVATION_MODULE, + "observe_adapter_sources_freshness", + ), )? .call((), Some(&options)) { @@ -332,8 +334,11 @@ fn configured_observation<'py>( options.set_item("connection", &observer.connection)?; options.set_item("source", source)?; options.set_item("observed_at", &observer.observed_at)?; - match freshness_function(py, ("observation", "observe_configured_source_freshness"))? - .call((), Some(&options)) + match freshness_function( + py, + (OBSERVATION_MODULE, "observe_configured_source_freshness"), + )? + .call((), Some(&options)) { Ok(observation) => Ok(Ok(observation)), Err(error) => Ok(Err(target_result( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 6fe5dbe9fc..eb83130059 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -28,18 +28,16 @@ const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; -const RUNTIME_PACKAGE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime"; +const PLANNING_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.planning"; +const EXECUTION_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.execution"; +const COMPILATION_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.compilation"; +const CONNECTIONS_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.connections"; const COMMAND_NAME: &str = "janitor"; fn imported<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { py.import(module)?.getattr(name) } -/// One of the janitor runtime phases, which reach the adapter and executor. -fn runtime<'py>(py: Python<'py>, (module, name): (&str, &str)) -> PyResult> { - imported(py, &format!("{RUNTIME_PACKAGE}.{module}"), name) -} - fn keywords<'py>( py: Python<'py>, items: &[(&str, &Bound<'py, PyAny>)], @@ -192,7 +190,7 @@ fn clean( let style: CliStyle = CliStyle { use_color: invocation.getattr("use_color")?.extract()?, }; - let planning: Bound<'_, PyAny> = runtime(py, ("planning", "build_janitor_execution_plan"))? + let planning: Bound<'_, PyAny> = imported(py, PLANNING_MODULE, "build_janitor_execution_plan")? .call( (), Some(&keywords( @@ -227,17 +225,18 @@ fn clean( write(py, "Janitor cancelled.\n")?; return Ok(1); } - let result: Bound<'_, PyAny> = runtime(py, ("execution", "execute_janitor_cleanup"))?.call( - (), - Some(&keywords( - py, - &[ - ("compile_context", compile_context), - ("connection_context", connection_context), - ("planning_result", &planning), - ], - )?), - )?; + let result: Bound<'_, PyAny> = imported(py, EXECUTION_MODULE, "execute_janitor_cleanup")? + .call( + (), + Some(&keywords( + py, + &[ + ("compile_context", compile_context), + ("connection_context", connection_context), + ("planning_result", &planning), + ], + )?), + )?; write( py, &janitor_completion_text(&janitor_result_counts(&result)?, style), @@ -272,10 +271,10 @@ fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult = - runtime(py, ("compilation", "compile_janitor_project"))? + imported(py, COMPILATION_MODULE, "compile_janitor_project")? .call((), Some(&keywords(py, &[("invocation", &invocation)])?))?; let connection_context: Bound<'_, PyAny> = - runtime(py, ("connections", "connect_janitor_warehouse"))?.call( + imported(py, CONNECTIONS_MODULE, "connect_janitor_warehouse")?.call( (), Some(&keywords( py, @@ -292,7 +291,7 @@ fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult> = - runtime(py, ("connections", "close_janitor_warehouse")).and_then(|close| { + imported(py, CONNECTIONS_MODULE, "close_janitor_warehouse").and_then(|close| { close.call( (), Some(&keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 5efc9c3d26..8ae19635f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -28,7 +28,9 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const EFFECTIVE_PACKAGE: &str = "sqlbuild.compiler.compile.main"; +const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main.effective_target"; +const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; +const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; @@ -77,10 +79,6 @@ fn write(stream: &Bound<'_, PyAny>, value: &str) -> PyResult<()> { Ok(()) } -fn effective<'py>(py: Python<'py>, (module, name): (&str, &str)) -> PyResult> { - imported(py, &format!("{EFFECTIVE_PACKAGE}.{module}"), name) -} - fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult> { let project_dir: Bound<'py, PyAny> = match request.getattr("project_dir")? { directory if !directory.is_none() => directory, @@ -102,7 +100,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = request.getattr("selected_target")?; let target_config: Bound<'py, PyAny> = call_with( - &effective(py, ("effective_target", "build_effective_target_config"))?, + &imported(py, EFFECTIVE_TARGET_MODULE, "build_effective_target_config")?, &[ ("discovered_inputs", discovered.clone()), ("selected_target", selected_target.clone()), @@ -306,7 +304,11 @@ fn prepare<'py>( )?; check_reference_targets(invocation, (&adapter, &connection_config))?; let runtime: (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = call_with( - &effective(py, ("effective_runtime", "build_effective_runtime_config"))?, + &imported( + py, + EFFECTIVE_RUNTIME_MODULE, + "build_effective_runtime_config", + )?, &[ ("discovered_inputs", discovered.clone()), ("selected_target", request.getattr("selected_target")?), @@ -317,9 +319,10 @@ fn prepare<'py>( let requested: Bound<'py, PyAny> = request.getattr("concurrency")?; let concurrency: i64 = if requested.is_none() { call_with( - &effective( + &imported( py, - ("effective_settings", "build_effective_settings_config"), + EFFECTIVE_SETTINGS_MODULE, + "build_effective_settings_config", )?, &[("discovered_inputs", discovered.clone())], )? From ad4b9cd1c0916d99d4e4d792f987061ed57826ce Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:12:59 +0100 Subject: [PATCH 24/37] test: check native Python module paths resolve and are never built at runtime --- crates/README.md | 15 ++ scripts/native_module_paths/__init__.py | 1 + .../native_module_paths/_helpers/__init__.py | 1 + .../native_module_paths/_helpers/crates.py | 40 ++++++ .../native_module_paths/_helpers/resolve.py | 33 +++++ scripts/native_module_paths/_helpers/scan.py | 76 ++++++++++ scripts/native_module_paths/constants.py | 34 +++++ scripts/native_module_paths/models.py | 23 +++ .../scripts/native_module_paths/__init__.py | 1 + .../native_module_paths/_helpers/__init__.py | 1 + .../_helpers/_test_types.py | 15 ++ .../native_module_paths/_helpers/helpers.py | 9 ++ .../_helpers/test_module_paths.py | 136 ++++++++++++++++++ 13 files changed, 385 insertions(+) create mode 100644 scripts/native_module_paths/__init__.py create mode 100644 scripts/native_module_paths/_helpers/__init__.py create mode 100644 scripts/native_module_paths/_helpers/crates.py create mode 100644 scripts/native_module_paths/_helpers/resolve.py create mode 100644 scripts/native_module_paths/_helpers/scan.py create mode 100644 scripts/native_module_paths/constants.py create mode 100644 scripts/native_module_paths/models.py create mode 100644 tests/unit/scripts/native_module_paths/__init__.py create mode 100644 tests/unit/scripts/native_module_paths/_helpers/__init__.py create mode 100644 tests/unit/scripts/native_module_paths/_helpers/_test_types.py create mode 100644 tests/unit/scripts/native_module_paths/_helpers/helpers.py create mode 100644 tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py diff --git a/crates/README.md b/crates/README.md index b1f38540f6..3bfdbb0b5f 100644 --- a/crates/README.md +++ b/crates/README.md @@ -57,6 +57,21 @@ The same check fails when the workspace version, the `pyproject.toml` version an `fensu check` applies the Rust structure rules inside each crate: other crates use items through `main/` entry points, `models.rs`, `types.rs` and `constants.rs`, not through `_helpers/`. +## Python module paths + +`sqlbuild-python` and `sqlbuild-cli` import Python by module path strings. The repository test +`tests/unit/scripts/native_module_paths` (`scripts/native_module_paths`) keeps them honest: + +- every `"sqlbuild...."` literal must be a complete dotted path that imports as a module or + resolves as a module attribute; constants whose name contains `LOGGER` hold `logging` + logger names and are not imported; +- attributes are checked where a literal module, or a module constant defined in the same file, + is passed with a literal name to an import helper (`imported(py, MODULE, "name")`), to + `import(MODULE)?.getattr("name")`, or written as a `("sqlbuild.module", "name")` pair; +- module paths are never built at runtime: a `format!` or `concat!` template may not start with + `sqlbuild.` or with a `{..PACKAGE..}.`/`{..MODULE..}.` constant. Write the complete module + path as a constant instead. + ## Compiler The compiler runs discovery, rendering and analysis natively. Generic compiler stages do not diff --git a/scripts/native_module_paths/__init__.py b/scripts/native_module_paths/__init__.py new file mode 100644 index 0000000000..64eb0e51a9 --- /dev/null +++ b/scripts/native_module_paths/__init__.py @@ -0,0 +1 @@ +"""Native Rust-to-Python module path checks.""" diff --git a/scripts/native_module_paths/_helpers/__init__.py b/scripts/native_module_paths/_helpers/__init__.py new file mode 100644 index 0000000000..3f583f245c --- /dev/null +++ b/scripts/native_module_paths/_helpers/__init__.py @@ -0,0 +1 @@ +"""Native module path helpers.""" diff --git a/scripts/native_module_paths/_helpers/crates.py b/scripts/native_module_paths/_helpers/crates.py new file mode 100644 index 0000000000..00a9210d22 --- /dev/null +++ b/scripts/native_module_paths/_helpers/crates.py @@ -0,0 +1,40 @@ +"""Check every Python module path the native crates import by string.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.native_module_paths._helpers.resolve import unresolved_reference_error +from scripts.native_module_paths._helpers.scan import scan_native_source +from scripts.native_module_paths.constants import RUST_SOURCE_GLOB, SCANNED_SOURCE_ROOTS +from scripts.native_module_paths.models import ModulePathReference, NativeModulePathScan + + +def scan_native_crates(*, root: Path) -> NativeModulePathScan: + """Scan every Rust file of the crates that call Python, in path order.""" + references: list[ModulePathReference] = [] + logger_names: set[str] = set() + violations: list[str] = [] + for source_root in SCANNED_SOURCE_ROOTS: + for path in sorted((root / source_root).rglob(RUST_SOURCE_GLOB)): + scan: NativeModulePathScan = scan_native_source( + source=path.read_text(encoding="utf-8"), label=path.relative_to(root).as_posix() + ) + references.extend(scan.references) + logger_names.update(scan.logger_names) + violations.extend(scan.violations) + return NativeModulePathScan( + references=tuple(references), + logger_names=tuple(sorted(logger_names)), + violations=tuple(violations), + ) + + +def native_module_path_errors(*, scan: NativeModulePathScan) -> tuple[str, ...]: + """Every policy violation, then every reference that does not resolve.""" + unresolved: list[str] = [] + for reference in scan.references: + error: str | None = unresolved_reference_error(reference=reference) + if error is not None: + unresolved.append(error) + return scan.violations + tuple(unresolved) diff --git a/scripts/native_module_paths/_helpers/resolve.py b/scripts/native_module_paths/_helpers/resolve.py new file mode 100644 index 0000000000..0f9cc6619b --- /dev/null +++ b/scripts/native_module_paths/_helpers/resolve.py @@ -0,0 +1,33 @@ +"""Resolve module paths found in native source against the importable Python package.""" + +from __future__ import annotations + +import importlib + +from scripts.native_module_paths.models import ModulePathReference + + +def unresolved_reference_error(*, reference: ModulePathReference) -> str | None: + """Why the reference does not resolve to a module, module attribute or class member.""" + target: object | None = _resolved(path=reference.path) + if target is None: + return f"{reference.location}: '{reference.path}' is not an importable module or attribute." + if reference.attribute is not None and not hasattr(target, reference.attribute): + return f"{reference.location}: '{reference.path}' has no attribute '{reference.attribute}'." + return None + + +def _resolved(*, path: str) -> object | None: + """The longest importable module prefix of `path`, then its remaining attributes.""" + parts: list[str] = path.split(".") + for split in range(len(parts), 0, -1): + try: + target: object = importlib.import_module(".".join(parts[:split])) + except ImportError: + continue + for attribute in parts[split:]: + if not hasattr(target, attribute): + return None + target = getattr(target, attribute) + return target + return None diff --git a/scripts/native_module_paths/_helpers/scan.py b/scripts/native_module_paths/_helpers/scan.py new file mode 100644 index 0000000000..e830a6fc39 --- /dev/null +++ b/scripts/native_module_paths/_helpers/scan.py @@ -0,0 +1,76 @@ +"""Find the Python module paths native Rust source imports by string.""" + +from __future__ import annotations + +from scripts.native_module_paths.constants import ( + DOTTED_PATH, + DYNAMIC_MODULE_PATHS, + IMPORT_GETATTR_CHAIN, + IMPORT_HELPER_CALL, + LITERAL_MODULE_ATTRIBUTE_PAIR, + LOGGER_CONSTANT_NAME, + PACKAGE_LITERAL, + STRING_CONSTANT, +) +from scripts.native_module_paths.models import ModulePathReference, NativeModulePathScan + + +def scan_native_source(*, source: str, label: str) -> NativeModulePathScan: + """Return the module paths, module attributes and dynamic module paths in one Rust file.""" + constants: dict[str, str] = { + match.group(1): match.group(2) for match in STRING_CONSTANT.finditer(source) + } + logger_names: frozenset[str] = frozenset( + value for name, value in constants.items() if LOGGER_CONSTANT_NAME.search(name) + ) + references: list[ModulePathReference] = [] + violations: list[str] = [] + for match in PACKAGE_LITERAL.finditer(source): + path: str = match.group(1) + location: str = _location(source=source, label=label, offset=match.start()) + if DOTTED_PATH.fullmatch(path) is None: + violations.append(f"{location}: '{path}' is not a complete dotted module path.") + elif path not in logger_names: + references.append(ModulePathReference(location=location, path=path)) + for pattern in (IMPORT_HELPER_CALL, IMPORT_GETATTR_CHAIN): + for match in pattern.finditer(source): + module: str | None = _module_argument(argument=match.group(1), constants=constants) + if module is not None and module not in logger_names: + references.append( + ModulePathReference( + location=_location(source=source, label=label, offset=match.start()), + path=module, + attribute=match.group(2), + ) + ) + for match in LITERAL_MODULE_ATTRIBUTE_PAIR.finditer(source): + references.append( + ModulePathReference( + location=_location(source=source, label=label, offset=match.start()), + path=match.group(1), + attribute=match.group(2), + ) + ) + for pattern in DYNAMIC_MODULE_PATHS: + violations.extend( + f"{_location(source=source, label=label, offset=match.start())}: module path is " + "built at runtime; use a complete literal module path." + for match in pattern.finditer(source) + ) + return NativeModulePathScan( + references=tuple(references), + logger_names=tuple(sorted(logger_names)), + violations=tuple(violations), + ) + + +def _module_argument(*, argument: str, constants: dict[str, str]) -> str | None: + """The module a helper argument names: a literal, or a same-file package constant.""" + if argument.startswith('"'): + return argument.strip('"') + return constants.get(argument) + + +def _location(*, source: str, label: str, offset: int) -> str: + line: int = source.count("\n", 0, offset) + 1 + return f"{label}:{line}" diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py new file mode 100644 index 0000000000..7cf304ae84 --- /dev/null +++ b/scripts/native_module_paths/constants.py @@ -0,0 +1,34 @@ +"""Native module path policy constants; crates/README.md describes the rules.""" + +import re + +SCANNED_SOURCE_ROOTS: tuple[str, ...] = ("crates/sqlbuild-python/src", "crates/sqlbuild-cli/src") +RUST_SOURCE_GLOB: str = "*.rs" +PACKAGE_LITERAL: re.Pattern[str] = re.compile(r'"(sqlbuild\.[^"\\]*)"') +DOTTED_PATH: re.Pattern[str] = re.compile(r"sqlbuild(?:\.[A-Za-z_][A-Za-z0-9_]*)+") +STRING_CONSTANT: re.Pattern[str] = re.compile( + r"\bconst\s+([A-Z][A-Z0-9_]*)\s*:\s*&(?:'static\s+)?str\s*=\s*\"(sqlbuild\.[^\"\\]*)\"\s*;" +) +LOGGER_CONSTANT_NAME: re.Pattern[str] = re.compile(r"LOGGER") +ATTRIBUTE_NAME: str = r"\"([A-Za-z_][A-Za-z0-9_]*)\"" +MODULE_ARGUMENT: str = r"([A-Z][A-Z0-9_]*|\"sqlbuild\.[^\"\\]*\")" +IMPORT_HELPER_CALL: re.Pattern[str] = re.compile( + r"\b[a-z_][a-z0-9_]*\(\s*py\s*,\s*" + + MODULE_ARGUMENT + + r"\s*,\s*" + + ATTRIBUTE_NAME + + r"\s*,?\s*\)" +) +IMPORT_GETATTR_CHAIN: re.Pattern[str] = re.compile( + r"\bimport\(\s*" + MODULE_ARGUMENT + r"\s*\)\?\s*\.getattr\(\s*" + ATTRIBUTE_NAME + r"\s*\)" +) +LITERAL_MODULE_ATTRIBUTE_PAIR: re.Pattern[str] = re.compile( + r"\(\s*\"(sqlbuild\.[^\"\\]*)\"\s*,\s*" + ATTRIBUTE_NAME + r"\s*,?\s*\)" +) +DYNAMIC_MODULE_PATHS: tuple[re.Pattern[str], ...] = ( + re.compile(r"\b(?:format|concat)!\(\s*\"\{[A-Z0-9_]*(?:PACKAGE|MODULE)[A-Z0-9_]*\}\."), + re.compile( + r"\b(?:format|concat)!\(\s*\"\{\}\.[^\"]*\"\s*,\s*[A-Z0-9_]*(?:PACKAGE|MODULE)[A-Z0-9_]*\b" + ), + re.compile(r"\b(?:format|concat)!\(\s*\"sqlbuild\."), +) diff --git a/scripts/native_module_paths/models.py b/scripts/native_module_paths/models.py new file mode 100644 index 0000000000..9b20922809 --- /dev/null +++ b/scripts/native_module_paths/models.py @@ -0,0 +1,23 @@ +"""Native module path scan models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ModulePathReference: + """One Python module path, or module and attribute, that native source imports.""" + + location: str + path: str + attribute: str | None = None + + +@dataclass(frozen=True) +class NativeModulePathScan: + """What a scan of native source found: checked references and policy violations.""" + + references: tuple[ModulePathReference, ...] + logger_names: tuple[str, ...] + violations: tuple[str, ...] diff --git a/tests/unit/scripts/native_module_paths/__init__.py b/tests/unit/scripts/native_module_paths/__init__.py new file mode 100644 index 0000000000..e017e10480 --- /dev/null +++ b/tests/unit/scripts/native_module_paths/__init__.py @@ -0,0 +1 @@ +"""Native module path tooling tests.""" diff --git a/tests/unit/scripts/native_module_paths/_helpers/__init__.py b/tests/unit/scripts/native_module_paths/_helpers/__init__.py new file mode 100644 index 0000000000..d54a5ed796 --- /dev/null +++ b/tests/unit/scripts/native_module_paths/_helpers/__init__.py @@ -0,0 +1 @@ +"""Native module path helper tests.""" diff --git a/tests/unit/scripts/native_module_paths/_helpers/_test_types.py b/tests/unit/scripts/native_module_paths/_helpers/_test_types.py new file mode 100644 index 0000000000..db04ab1cdc --- /dev/null +++ b/tests/unit/scripts/native_module_paths/_helpers/_test_types.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeSourceModulePathTestCase: + description: str + source: str + expected_errors: tuple[str, ...] + + +@dataclass(frozen=True) +class RepositoryModulePathTestCase: + description: str + expected_errors: tuple[str, ...] + expected_logger_names: tuple[str, ...] diff --git a/tests/unit/scripts/native_module_paths/_helpers/helpers.py b/tests/unit/scripts/native_module_paths/_helpers/helpers.py new file mode 100644 index 0000000000..f8c3865cc3 --- /dev/null +++ b/tests/unit/scripts/native_module_paths/_helpers/helpers.py @@ -0,0 +1,9 @@ +from scripts.native_module_paths._helpers.crates import native_module_path_errors +from scripts.native_module_paths._helpers.scan import scan_native_source + +SOURCE_LABEL: str = "lib.rs" + + +def native_source_errors(source: str) -> tuple[str, ...]: + """Scan one synthetic Rust file and resolve what it imports.""" + return native_module_path_errors(scan=scan_native_source(source=source, label=SOURCE_LABEL)) diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py new file mode 100644 index 0000000000..36668c4da2 --- /dev/null +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -0,0 +1,136 @@ +from pathlib import Path + +import pytest + +from scripts.native_module_paths._helpers.crates import ( + native_module_path_errors, + scan_native_crates, +) +from scripts.native_module_paths.models import NativeModulePathScan +from tests.unit.scripts.native_module_paths._helpers._test_types import ( + NativeSourceModulePathTestCase, + RepositoryModulePathTestCase, +) +from tests.unit.scripts.native_module_paths._helpers.helpers import native_source_errors + +EXCEPTIONS_MODULE: str = "sqlbuild.cli.commands.exceptions" + + +@pytest.mark.parametrize( + "test_case", + [ + NativeSourceModulePathTestCase( + description="module constant and helper attribute resolve", + source=( + f'const ERRORS_MODULE: &str = "{EXCEPTIONS_MODULE}";\n' + 'let error = imported(py, ERRORS_MODULE, "CliUserError")?;\n' + ), + expected_errors=(), + ), + NativeSourceModulePathTestCase( + description="literal names a module attribute", + source=f'let error = py.import("sqlbuild")?;\nlet name = "{EXCEPTIONS_MODULE}.CliUserError";\n', + expected_errors=(), + ), + NativeSourceModulePathTestCase( + description="logger constant is not imported", + source='const LOGGER_NAME: &str = "sqlbuild.compile";\n', + expected_errors=(), + ), + NativeSourceModulePathTestCase( + description="relation names are formatted from lower-case values", + source=( + 'let relation = format!("{schema}.{table}");\n' + 'let qualified = format!("{}.{}", text_of(schema)?, text_of(&alias)?);\n' + 'let temporary = format!(".{CACHE_FILE_NAME}.{}.tmp", id);\n' + ), + expected_errors=(), + ), + NativeSourceModulePathTestCase( + description="unknown module literal", + source='const MISSING_MODULE: &str = "sqlbuild.cli.no_such_module";\n', + expected_errors=( + "lib.rs:1: 'sqlbuild.cli.no_such_module' is not an importable module or attribute.", + ), + ), + NativeSourceModulePathTestCase( + description="helper attribute missing from a module constant", + source=( + f'const ERRORS_MODULE: &str =\n "{EXCEPTIONS_MODULE}";\n' + 'let error = imported(\n py,\n ERRORS_MODULE,\n "NoSuchError",\n)?;\n' + ), + expected_errors=(f"lib.rs:3: '{EXCEPTIONS_MODULE}' has no attribute 'NoSuchError'.",), + ), + NativeSourceModulePathTestCase( + description="getattr missing from an imported literal", + source=f'let error = py\n .import("{EXCEPTIONS_MODULE}")?\n .getattr("NoSuchError")?;\n', + expected_errors=(f"lib.rs:2: '{EXCEPTIONS_MODULE}' has no attribute 'NoSuchError'.",), + ), + NativeSourceModulePathTestCase( + description="literal module and attribute pair missing the attribute", + source=f'const CALLS: [(&str, &str); 1] = [("{EXCEPTIONS_MODULE}", "NoSuchError")];\n', + expected_errors=(f"lib.rs:1: '{EXCEPTIONS_MODULE}' has no attribute 'NoSuchError'.",), + ), + NativeSourceModulePathTestCase( + description="package constant joined with a module name at runtime", + source='let module = format!("{RUNTIME_PACKAGE}.{module}");\n', + expected_errors=( + "lib.rs:1: module path is built at runtime; use a complete literal module path.", + ), + ), + NativeSourceModulePathTestCase( + description="module constant passed positionally to a joined template", + source='let module = format!(\n "{}.{}",\n EFFECTIVE_MODULE,\n module,\n);\n', + expected_errors=( + "lib.rs:1: module path is built at runtime; use a complete literal module path.", + ), + ), + NativeSourceModulePathTestCase( + description="package prefix completed at runtime", + source='let module = format!("sqlbuild.compiler.{module}");\n', + expected_errors=( + "lib.rs:1: 'sqlbuild.compiler.{module}' is not a complete dotted module path.", + "lib.rs:1: module path is built at runtime; use a complete literal module path.", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_native_source_when_checking_module_paths_then_reports_errors( + test_case: NativeSourceModulePathTestCase, +) -> None: + assert native_source_errors(test_case.source) == test_case.expected_errors + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryModulePathTestCase( + description="native crates import only existing Python modules and attributes", + expected_errors=(), + expected_logger_names=( + "sqlbuild.cli", + "sqlbuild.cli.commands._helpers.cost.collection", + "sqlbuild.cli.commands.classes.prepared_compile_artifacts", + "sqlbuild.compile", + "sqlbuild.fingerprints", + "sqlbuild.lineage", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_native_crates_when_checking_then_module_paths_resolve( + test_case: RepositoryModulePathTestCase, +) -> None: + repository: Path = Path(__file__).resolve().parents[5] + + scan: NativeModulePathScan = scan_native_crates(root=repository) + + assert scan.references + assert scan.logger_names == test_case.expected_logger_names + assert native_module_path_errors(scan=scan) == test_case.expected_errors + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 6b3106ffcb70a9aa285d2231b48c07c908a90a82 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:15:31 +0100 Subject: [PATCH 25/37] feat: run sqb test natively --- crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- crates/sqlbuild-core/src/lib.rs | 1 + .../src/test_command/_helpers/coded_error.rs | 26 + .../src/test_command/_helpers/mod.rs | 1 + .../src/test_command/constants.rs | 9 + .../src/test_command/main/mod.rs | 8 + .../main/parameterized_test_label.rs | 31 + .../main/render_test_completion.rs | 70 ++ .../main/render_test_inspection.rs | 55 ++ .../test_command/main/render_test_ready.rs | 17 + .../src/test_command/main/test_exit_code.rs | 7 + .../src/test_command/main/test_group_name.rs | 13 + .../src/test_command/main/test_name_width.rs | 15 + .../test_command/main/test_outcome_status.rs | 11 + crates/sqlbuild-core/src/test_command/mod.rs | 9 + .../sqlbuild-core/src/test_command/models.rs | 40 + .../sqlbuild-core/src/test_command/tests.rs | 6 + .../src/test_command/tests/helpers.rs | 11 + .../test_command/tests/test_test_command.rs | 258 ++++++ .../src/test_command/tests/test_types.rs | 51 ++ .../src/bindings/_helpers/cli/registration.rs | 3 +- .../src/bindings/_helpers/mod.rs | 1 + .../bindings/_helpers/sql_test_command/mod.rs | 2 + .../_helpers/sql_test_command/registration.rs | 9 + .../_helpers/sql_test_command/test_command.rs | 755 ++++++++++++++++++ src/sqlbuild/_native.pyi | 16 + .../cli/commands/_helpers/test/execution.py | 208 ----- .../cli/commands/_helpers/test/inspection.py | 64 -- .../cli/commands/_helpers/test/invocation.py | 18 - .../cli/commands/_helpers/test/outputs.py | 98 --- .../cli/commands/_helpers/test/planning.py | 77 -- .../commands/_helpers/test/sql_progress.py | 103 --- .../classes/build_progress_callbacks.py | 2 +- .../cli/commands/main/execution/_test.py | 70 -- src/sqlbuild/cli/commands/models.py | 14 - .../commands/{sql_test.py => _sql_test.py} | 0 .../runtime/test_command_invocation.py | 7 +- .../cli/commands/main/test_load_command.py | 4 +- .../commands/shared/_helpers/test_progress.py | 8 +- .../test_execution_result_document.py | 3 +- .../sql_test_assembly/test_plan_test.py | 2 +- .../main/test_static_command_planning.py | 2 +- 42 files changed, 1441 insertions(+), 668 deletions(-) create mode 100644 crates/sqlbuild-core/src/test_command/_helpers/coded_error.rs create mode 100644 crates/sqlbuild-core/src/test_command/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/test_command/constants.rs create mode 100644 crates/sqlbuild-core/src/test_command/main/mod.rs create mode 100644 crates/sqlbuild-core/src/test_command/main/parameterized_test_label.rs create mode 100644 crates/sqlbuild-core/src/test_command/main/render_test_completion.rs create mode 100644 crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs create mode 100644 crates/sqlbuild-core/src/test_command/main/render_test_ready.rs create mode 100644 crates/sqlbuild-core/src/test_command/main/test_exit_code.rs create mode 100644 crates/sqlbuild-core/src/test_command/main/test_group_name.rs create mode 100644 crates/sqlbuild-core/src/test_command/main/test_name_width.rs create mode 100644 crates/sqlbuild-core/src/test_command/main/test_outcome_status.rs create mode 100644 crates/sqlbuild-core/src/test_command/mod.rs create mode 100644 crates/sqlbuild-core/src/test_command/models.rs create mode 100644 crates/sqlbuild-core/src/test_command/tests.rs create mode 100644 crates/sqlbuild-core/src/test_command/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/test_command/tests/test_test_command.rs create mode 100644 crates/sqlbuild-core/src/test_command/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/test/execution.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/test/inspection.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/test/invocation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/test/outputs.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/test/planning.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/test/sql_progress.py delete mode 100644 src/sqlbuild/cli/commands/main/execution/_test.py rename src/sqlbuild/compiler/planner/main/commands/{sql_test.py => _sql_test.py} (100%) diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index f2398f3505..c639afd309 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -240,9 +240,9 @@ const TEST_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const TEST_ROUTE: Route = routed( "run_test", - ("sqlbuild.cli.commands.main.execution._test", "run_test"), + ("sqlbuild._native", "run_test_command"), &TEST_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const CHECK_FIELDS: [Field; 9] = [ diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index c5b22fb902..f5423d1dbb 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -19,6 +19,7 @@ pub mod posix_path; pub mod progress; pub mod redaction; pub mod terminal_events; +pub mod test_command; pub mod text; #[cfg(test)] diff --git a/crates/sqlbuild-core/src/test_command/_helpers/coded_error.rs b/crates/sqlbuild-core/src/test_command/_helpers/coded_error.rs new file mode 100644 index 0000000000..8b547ec171 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/_helpers/coded_error.rs @@ -0,0 +1,26 @@ +//! A result-stored coded error, as CLI expected errors render it. + +use crate::cli_text::models::CliStyle; + +/// `[CODE] message`, later message lines muted, then the help line when there is one. +pub(crate) fn coded_error( + (code, message, help): (&str, &str, Option<&str>), + style: CliStyle, +) -> String { + let mut lines = message.split('\n'); + let first: &str = lines.next().unwrap_or(""); + let rest: Vec = lines.map(|line| style.muted(line)).collect(); + let rendered_message: String = if rest.is_empty() { + message.to_owned() + } else { + format!("{first}\n{}", rest.join("\n")) + }; + let mut rendered: String = format!( + "{} {rendered_message}", + style.error_strong(&format!("[{code}]")) + ); + if let Some(help) = help { + rendered = format!("{rendered}\n {} {help}", style.muted("= help:")); + } + rendered +} diff --git a/crates/sqlbuild-core/src/test_command/_helpers/mod.rs b/crates/sqlbuild-core/src/test_command/_helpers/mod.rs new file mode 100644 index 0000000000..bb61fa37f5 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod coded_error; diff --git a/crates/sqlbuild-core/src/test_command/constants.rs b/crates/sqlbuild-core/src/test_command/constants.rs new file mode 100644 index 0000000000..883fcc2cf2 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/constants.rs @@ -0,0 +1,9 @@ +//! Test output labels and widths. + +pub(crate) const ERROR_LABEL: &str = "error"; +pub(crate) const ERROR_LABEL_WIDTH: usize = 10; +pub(crate) const DEFAULT_ERROR: &str = "test failed"; +pub(crate) const UNKNOWN_GROUP: &str = "(unknown)"; +pub(crate) const NAME_MIN_WIDTH: usize = 50; +pub(crate) const CHILD_NAME_INDENT: usize = 4; +pub(crate) const ERROR_SEVERITY: &str = "error"; diff --git a/crates/sqlbuild-core/src/test_command/main/mod.rs b/crates/sqlbuild-core/src/test_command/main/mod.rs new file mode 100644 index 0000000000..55ae286b69 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/mod.rs @@ -0,0 +1,8 @@ +pub mod parameterized_test_label; +pub mod render_test_completion; +pub mod render_test_inspection; +pub mod render_test_ready; +pub mod test_exit_code; +pub mod test_group_name; +pub mod test_name_width; +pub mod test_outcome_status; diff --git a/crates/sqlbuild-core/src/test_command/main/parameterized_test_label.rs b/crates/sqlbuild-core/src/test_command/main/parameterized_test_label.rs new file mode 100644 index 0000000000..5728eab8f4 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/parameterized_test_label.rs @@ -0,0 +1,31 @@ +//! The label of a test row. + +use crate::test_command::models::TestParameter; + +/// The name, with source path and typed values only for parameterized cases. +#[must_use] +pub fn parameterized_test_label( + name: &str, + source_path: Option<&str>, + parameters: &[TestParameter], +) -> String { + let Some(source_path) = source_path else { + return name.to_owned(); + }; + if parameters.is_empty() { + return name.to_owned(); + } + let rendered: Vec = parameters + .iter() + .map(|parameter| { + format!( + "{}:{}{}={}", + parameter.name, + parameter.value_type, + if parameter.nullable { "?" } else { "" }, + parameter.rendered + ) + }) + .collect(); + format!("{name} ({source_path}; {})", rendered.join(", ")) +} diff --git a/crates/sqlbuild-core/src/test_command/main/render_test_completion.rs b/crates/sqlbuild-core/src/test_command/main/render_test_completion.rs new file mode 100644 index 0000000000..8718d229b8 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/render_test_completion.rs @@ -0,0 +1,70 @@ +//! The test summary footer and failure details. + +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, SummaryCount}; +use crate::test_command::_helpers::coded_error::coded_error; +use crate::test_command::constants::{DEFAULT_ERROR, ERROR_LABEL, ERROR_LABEL_WIDTH}; +use crate::test_command::models::TestFailure; +use crate::text::main::python_splitlines::python_splitlines; + +fn count(label: &str, value: usize) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value: i64::try_from(value).unwrap_or(i64::MAX), + } +} + +fn failure_lines(failure: &TestFailure, style: CliStyle) -> Vec { + let message: &str = failure + .error_message + .as_deref() + .filter(|text| !text.is_empty()) + .unwrap_or(DEFAULT_ERROR); + let rendered: String = match &failure.error_code { + Some(code) => coded_error((code, message, failure.error_help.as_deref()), style), + None => message.to_owned(), + }; + let mut split: Vec<&str> = python_splitlines(&rendered); + if split.is_empty() { + split.push(&rendered); + } + let mut lines: Vec = vec![format!(" {}", failure.label)]; + let label: String = style.error_muted(&format!("{ERROR_LABEL: String { + let summary: String = format_summary_footer( + style, + &[ + count("PASS", passed), + count("FAIL", total - passed), + count("TOTAL", total), + ], + None, + ); + let mut rendered: String = format!("\n{summary}"); + if !failures.is_empty() { + let mut lines: Vec = vec![style.error_strong("Failures:"), String::new()]; + for failure in failures { + lines.extend(failure_lines(failure, style)); + } + rendered = format!("{rendered}\n\n{}", lines.join("\n").trim_end()); + } + format!("{rendered}\n") +} diff --git a/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs b/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs new file mode 100644 index 0000000000..c79d505b15 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs @@ -0,0 +1,55 @@ +//! `sqb test --inspect`: the resolved test plan without executing it. + +use crate::test_command::constants::ERROR_SEVERITY; +use crate::test_command::models::{TestInspectionEntry, TestPlanWarning}; + +fn names(label: &str, values: &[String]) -> String { + let shown: String = if values.is_empty() { + "none".to_owned() + } else { + values.join(", ") + }; + format!(" {label}: {shown}\n") +} + +/// Each test's fixture boundaries and model chain, warnings, the outcome line, and the exit code. +#[must_use] +pub fn render_test_inspection( + entries: &[TestInspectionEntry], + warnings: &[TestPlanWarning], +) -> (String, i32) { + let mut text: String = "\nResolved test plan\n\n".to_owned(); + for entry in entries { + text.push_str(&format!("{}\n", entry.label)); + for (label, values) in [ + ("mocked sources", &entry.mocked_sources), + ("mocked refs", &entry.mocked_refs), + ("mocked seeds", &entry.mocked_seeds), + ("mocked dbt refs", &entry.mocked_dbt_refs), + ("mocked table functions", &entry.mocked_table_functions), + ("real models", &entry.real_models), + ("expected models", &entry.expected_models), + ] { + text.push_str(&names(label, values)); + } + for mock in &entry.mocked_refs { + text.push_str(&format!( + " boundary: {mock} is replaced by __ref__{mock}\n" + )); + } + text.push('\n'); + } + let errors: usize = warnings + .iter() + .filter(|warning| warning.severity == ERROR_SEVERITY) + .count(); + for warning in warnings { + text.push_str(&format!(" {}: {}\n", warning.severity, warning.message)); + } + text.push_str(&format!( + "Test plan inspection {}: {} selected, {errors} errors.\n", + if errors > 0 { "failed" } else { "complete" }, + entries.len() + )); + (text, i32::from(errors > 0)) +} diff --git a/crates/sqlbuild-core/src/test_command/main/render_test_ready.rs b/crates/sqlbuild-core/src/test_command/main/render_test_ready.rs new file mode 100644 index 0000000000..4f248bd388 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/render_test_ready.rs @@ -0,0 +1,17 @@ +//! The header announcing the selected tests. + +use crate::cli_text::main::format_surface_header::format_surface_header; +use crate::cli_text::models::CliStyle; + +/// A blank line, `Test ready selected, models`, and a blank line. +#[must_use] +pub fn render_test_ready((tests, models): (usize, usize), style: CliStyle) -> String { + format!( + "\n{}\n\n", + format_surface_header( + style, + "Test ready", + &format!("{tests} selected, {models} models") + ) + ) +} diff --git a/crates/sqlbuild-core/src/test_command/main/test_exit_code.rs b/crates/sqlbuild-core/src/test_command/main/test_exit_code.rs new file mode 100644 index 0000000000..601794de1b --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/test_exit_code.rs @@ -0,0 +1,7 @@ +//! The exit code of `sqb test`. + +/// 1 when any test did not pass, else 0. +#[must_use] +pub fn test_exit_code(passed: usize, total: usize) -> i32 { + i32::from(passed != total) +} diff --git a/crates/sqlbuild-core/src/test_command/main/test_group_name.rs b/crates/sqlbuild-core/src/test_command/main/test_group_name.rs new file mode 100644 index 0000000000..a6fdb63b18 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/test_group_name.rs @@ -0,0 +1,13 @@ +//! The model a test's progress rows are grouped under. + +use crate::test_command::constants::UNKNOWN_GROUP; + +/// The first model with expected rows, else the first model, else `(unknown)`. +#[must_use] +pub fn test_group_name(chain: &[(String, bool)]) -> String { + chain + .iter() + .find(|(_, expected)| *expected) + .or_else(|| chain.first()) + .map_or_else(|| UNKNOWN_GROUP.to_owned(), |(model, _)| model.clone()) +} diff --git a/crates/sqlbuild-core/src/test_command/main/test_name_width.rs b/crates/sqlbuild-core/src/test_command/main/test_name_width.rs new file mode 100644 index 0000000000..94a73a5192 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/test_name_width.rs @@ -0,0 +1,15 @@ +//! One shared width for test rows and their nested expectation rows. + +use crate::cli_text::main::resolve_name_column_width::resolve_name_column_width; +use crate::test_command::constants::{CHILD_NAME_INDENT, NAME_MIN_WIDTH}; + +/// The test label column width, widened for the longest indented child name. +#[must_use] +pub fn test_name_width(labels: &[String], child_names: &[String]) -> usize { + let child: usize = child_names + .iter() + .map(|name| name.chars().count()) + .max() + .unwrap_or(0); + resolve_name_column_width(labels, NAME_MIN_WIDTH).max(child + CHILD_NAME_INDENT) +} diff --git a/crates/sqlbuild-core/src/test_command/main/test_outcome_status.rs b/crates/sqlbuild-core/src/test_command/main/test_outcome_status.rs new file mode 100644 index 0000000000..42941e1198 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/test_outcome_status.rs @@ -0,0 +1,11 @@ +//! The status word of a test outcome. + +/// `PASS`, `ERROR` for execution errors, else `FAIL`. +#[must_use] +pub fn test_outcome_status(outcome: &str) -> &'static str { + match outcome { + "pass" => "PASS", + "error" => "ERROR", + _ => "FAIL", + } +} diff --git a/crates/sqlbuild-core/src/test_command/mod.rs b/crates/sqlbuild-core/src/test_command/mod.rs new file mode 100644 index 0000000000..34acbbf1d3 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/mod.rs @@ -0,0 +1,9 @@ +//! `sqb test`: test labels, plan inspection, completion summary and exit code. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/test_command/models.rs b/crates/sqlbuild-core/src/test_command/models.rs new file mode 100644 index 0000000000..aa17cc236e --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/models.rs @@ -0,0 +1,40 @@ +//! SQL test plan entries and results as the command renders them. + +/// One typed parameter value of a parameterized test case. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TestParameter { + pub name: String, + pub value_type: String, + pub nullable: bool, + /// The value as compact ASCII JSON. + pub rendered: String, +} + +/// One selected test as `--inspect` shows it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestInspectionEntry { + pub label: String, + pub mocked_sources: Vec, + pub mocked_refs: Vec, + pub mocked_seeds: Vec, + pub mocked_dbt_refs: Vec, + pub mocked_table_functions: Vec, + pub real_models: Vec, + pub expected_models: Vec, +} + +/// One plan warning: its severity value and message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TestPlanWarning { + pub severity: String, + pub message: String, +} + +/// One test that did not pass. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestFailure { + pub label: String, + pub error_message: Option, + pub error_code: Option, + pub error_help: Option, +} diff --git a/crates/sqlbuild-core/src/test_command/tests.rs b/crates/sqlbuild-core/src/test_command/tests.rs new file mode 100644 index 0000000000..5d8555f9fb --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_test_command.rs"] +mod test_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/test_command/tests/helpers.rs b/crates/sqlbuild-core/src/test_command/tests/helpers.rs new file mode 100644 index 0000000000..dfabe0753f --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/tests/helpers.rs @@ -0,0 +1,11 @@ +use crate::test_command::models::TestParameter; + +/// A string parameter with its rendered JSON value. +pub(super) fn parameter(name: &str, nullable: bool, rendered: &str) -> TestParameter { + TestParameter { + name: name.to_owned(), + value_type: "string".to_owned(), + nullable, + rendered: rendered.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs b/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs new file mode 100644 index 0000000000..46b2e84661 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs @@ -0,0 +1,258 @@ +use crate::cli_text::models::CliStyle; +use crate::test_command::main::parameterized_test_label::parameterized_test_label; +use crate::test_command::main::render_test_completion::render_test_completion; +use crate::test_command::main::render_test_inspection::render_test_inspection; +use crate::test_command::main::render_test_ready::render_test_ready; +use crate::test_command::main::test_exit_code::test_exit_code; +use crate::test_command::main::test_group_name::test_group_name; +use crate::test_command::main::test_name_width::test_name_width; +use crate::test_command::main::test_outcome_status::test_outcome_status; +use crate::test_command::models::{TestFailure, TestInspectionEntry, TestPlanWarning}; +use crate::test_command::tests::helpers::parameter; +use crate::test_command::tests::test_types::{ + CompletionTestCase, GroupTestCase, InspectionTestCase, LabelTestCase, OutcomeTestCase, + ReadyTestCase, WidthTestCase, +}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_test_when_labelling_then_adds_source_and_values_only_for_parameterized_cases() { + let test_cases = [ + LabelTestCase { + description: "a plain test keeps its name", + source_path: Some("tests/unit/test_orders.sql"), + parameters: Vec::new(), + expected_label: "order_status", + }, + LabelTestCase { + description: "no source path keeps the name", + source_path: None, + parameters: vec![parameter("status", false, "\"open\"")], + expected_label: "order_status", + }, + LabelTestCase { + description: "parameters render typed values and nullability", + source_path: Some("tests/unit/test_orders.sql"), + parameters: vec![ + parameter("status", false, "\"open\""), + parameter("note", true, "null"), + ], + expected_label: "order_status (tests/unit/test_orders.sql; status:string=\"open\", note:string?=null)", + }, + ]; + + for test_case in test_cases { + assert_eq!( + parameterized_test_label("order_status", test_case.source_path, &test_case.parameters), + test_case.expected_label, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_chain_when_grouping_then_prefers_first_expected_model() { + let test_cases = [ + GroupTestCase { + description: "the first model with expected rows names the group", + chain: vec![("stg".to_owned(), false), ("orders".to_owned(), true)], + expected_group: "orders", + }, + GroupTestCase { + description: "without expected rows the first model names the group", + chain: vec![("stg".to_owned(), false), ("orders".to_owned(), false)], + expected_group: "stg", + }, + GroupTestCase { + description: "an empty chain is unknown", + chain: Vec::new(), + expected_group: "(unknown)", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_group_name(&test_case.chain), + test_case.expected_group, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_outcomes_when_rendering_status_then_distinguishes_errors_from_failures() { + let test_cases = [ + OutcomeTestCase { + description: "a pass is PASS", + outcome: "pass", + expected_status: "PASS", + }, + OutcomeTestCase { + description: "an execution error is ERROR", + outcome: "error", + expected_status: "ERROR", + }, + OutcomeTestCase { + description: "an assertion failure is FAIL", + outcome: "fail", + expected_status: "FAIL", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_outcome_status(test_case.outcome), + test_case.expected_status, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_names_when_resolving_width_then_widens_for_indented_children() { + let test_cases = [ + WidthTestCase { + description: "short names use the minimum width", + labels: vec!["orders_assert".to_owned()], + child_names: Vec::new(), + expected_width: 50, + }, + WidthTestCase { + description: "a long child name widens by its indent", + labels: vec!["orders_assert".to_owned()], + child_names: vec!["x".repeat(60)], + expected_width: 64, + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_name_width(&test_case.labels, &test_case.child_names), + test_case.expected_width, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_counts_when_rendering_ready_then_writes_surface_header() { + let test_cases = [ReadyTestCase { + description: "counts tests and models", + counts: (1, 2), + expected_text: "\nTest ready 1 selected, 2 models\n\n", + }]; + + for test_case in test_cases { + assert_eq!( + render_test_ready(test_case.counts, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_completing_then_renders_summary_failures_and_exit_code() { + let test_cases = [ + CompletionTestCase { + description: "all passing writes only the summary", + counts: (2, 2), + failures: Vec::new(), + expected_text: "\nPASS=2 FAIL=0 TOTAL=2\n", + expected_exit_code: 0, + }, + CompletionTestCase { + description: "a coded failure renders its code, muted lines and help", + counts: (0, 1), + failures: vec![TestFailure { + label: "orders_assert".to_owned(), + error_message: Some("first\nsecond".to_owned()), + error_code: Some("T003".to_owned()), + error_help: Some("fix it".to_owned()), + }], + expected_text: "\nPASS=0 FAIL=1 TOTAL=1\n\nFailures:\n\n orders_assert\n error [T003] first\n second\n = help: fix it\n", + expected_exit_code: 1, + }, + CompletionTestCase { + description: "an uncoded empty failure says the test failed", + counts: (1, 2), + failures: vec![TestFailure { + label: "orders_unique".to_owned(), + error_message: Some(String::new()), + ..TestFailure::default() + }], + expected_text: "\nPASS=1 FAIL=1 TOTAL=2\n\nFailures:\n\n orders_unique\n error test failed\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_test_completion(test_case.counts, &test_case.failures, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + assert_eq!( + test_exit_code(test_case.counts.0, test_case.counts.1), + test_case.expected_exit_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_plan_when_inspecting_then_lists_boundaries_and_counts_errors() { + let entry = TestInspectionEntry { + label: "order_status [open_case]".to_owned(), + mocked_sources: vec!["raw_orders".to_owned()], + mocked_refs: vec!["stg".to_owned()], + real_models: vec!["orders".to_owned()], + expected_models: vec!["orders".to_owned()], + ..TestInspectionEntry::default() + }; + let test_cases = [ + InspectionTestCase { + description: "nothing selected completes", + entries: Vec::new(), + warnings: Vec::new(), + expected_text: "\nResolved test plan\n\nTest plan inspection complete: 0 selected, 0 errors.\n", + expected_exit_code: 0, + }, + InspectionTestCase { + description: "an error warning fails the inspection", + entries: vec![entry], + warnings: vec![ + TestPlanWarning { + severity: "warning".to_owned(), + message: "careful".to_owned(), + }, + TestPlanWarning { + severity: "error".to_owned(), + message: "broken".to_owned(), + }, + ], + expected_text: "\nResolved test plan\n\norder_status [open_case]\n mocked sources: raw_orders\n mocked refs: stg\n mocked seeds: none\n mocked dbt refs: none\n mocked table functions: none\n real models: orders\n expected models: orders\n boundary: stg is replaced by __ref__stg\n\n warning: careful\n error: broken\nTest plan inspection failed: 1 selected, 1 errors.\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_test_inspection(&test_case.entries, &test_case.warnings), + ( + test_case.expected_text.to_owned(), + test_case.expected_exit_code + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/test_command/tests/test_types.rs b/crates/sqlbuild-core/src/test_command/tests/test_types.rs new file mode 100644 index 0000000000..96c173c1d7 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/tests/test_types.rs @@ -0,0 +1,51 @@ +use crate::test_command::models::{ + TestFailure, TestInspectionEntry, TestParameter, TestPlanWarning, +}; + +pub(super) struct LabelTestCase { + pub(super) description: &'static str, + pub(super) source_path: Option<&'static str>, + pub(super) parameters: Vec, + pub(super) expected_label: &'static str, +} + +pub(super) struct GroupTestCase { + pub(super) description: &'static str, + pub(super) chain: Vec<(String, bool)>, + pub(super) expected_group: &'static str, +} + +pub(super) struct CompletionTestCase { + pub(super) description: &'static str, + pub(super) counts: (usize, usize), + pub(super) failures: Vec, + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} + +pub(super) struct InspectionTestCase { + pub(super) description: &'static str, + pub(super) entries: Vec, + pub(super) warnings: Vec, + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} + +pub(super) struct OutcomeTestCase { + pub(super) description: &'static str, + pub(super) outcome: &'static str, + pub(super) expected_status: &'static str, +} + +pub(super) struct WidthTestCase { + pub(super) description: &'static str, + pub(super) labels: Vec, + pub(super) child_names: Vec, + pub(super) expected_width: usize, +} + +pub(super) struct ReadyTestCase { + pub(super) description: &'static str, + pub(super) counts: (usize, usize), + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index b560bc368b..850e9ec734 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; use crate::bindings::_helpers::{ check, clone, command_runtime, cost_report, dbt_interop, debug_command, diff_report, freshness, - janitor, load, presentation_text, result_output, workspace_commands, + janitor, load, presentation_text, result_output, sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -27,5 +27,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { presentation_text::registration::register(module)?; workspace_commands::registration::register(module)?; check::registration::register(module)?; + sql_test_command::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 27686c92dc..5d348fc503 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -46,6 +46,7 @@ pub(crate) mod rules; pub(crate) mod scopes; pub(crate) mod semantic_checks; pub(crate) mod semantic_shapes; +pub(crate) mod sql_test_command; pub(crate) mod sql_test_glue; pub(crate) mod sqltext; pub(crate) mod state_rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs new file mode 100644 index 0000000000..1e38419e09 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod registration; +pub(crate) mod test_command; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs new file mode 100644 index 0000000000..daf61f4830 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs @@ -0,0 +1,9 @@ +//! Register the native `sqb test` command and its progress labels. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::sql_test_command::test_command; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + test_command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs new file mode 100644 index 0000000000..fc9b4685a8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -0,0 +1,755 @@ +//! `sqb test`: compile the selected SQL unit tests, inspect or run them, and report. + +use std::collections::{BTreeSet, HashMap}; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; +use sqlbuild_core::test_command::main::parameterized_test_label::parameterized_test_label; +use sqlbuild_core::test_command::main::render_test_completion::render_test_completion; +use sqlbuild_core::test_command::main::render_test_inspection::render_test_inspection; +use sqlbuild_core::test_command::main::render_test_ready::render_test_ready; +use sqlbuild_core::test_command::main::test_exit_code::test_exit_code; +use sqlbuild_core::test_command::main::test_group_name::test_group_name; +use sqlbuild_core::test_command::main::test_name_width::test_name_width; +use sqlbuild_core::test_command::main::test_outcome_status::test_outcome_status; +use sqlbuild_core::test_command::models::{ + TestFailure, TestInspectionEntry, TestParameter, TestPlanWarning, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::result_output::integration_values::json_value; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main.static_command"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; +const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; +const NESTED_PROGRESS_MODULE: &str = + "sqlbuild.cli.progress.classes.nested_command_progress_callbacks"; +const PROGRESS_MODELS: &str = "sqlbuild.cli.progress.models"; +const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; +const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; +const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; +const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main.resource_id"; +const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_test_runtime_target"; +const DECIMAL_KIND: &str = "decimal"; +const PASS_OUTCOME: &str = "pass"; +const EXPECT_LABEL: &str = "expect"; +const UNKNOWN_GROUP: &str = "(unknown)"; +const EXECUTION_COMMAND: &str = "sqb test"; +const UNKNOWN_CASE_CODE: &str = "S008"; + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn strings(values: Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +/// A payload as `json.dumps(payload, ensure_ascii=True, separators=(",", ":"))`. +fn compact_json(payload: &Bound<'_, PyAny>) -> PyResult { + let Some(value) = json_value(payload)? else { + let options: Bound<'_, PyDict> = PyDict::new(payload.py()); + options.set_item("ensure_ascii", true)?; + options.set_item("separators", (",", ":"))?; + return payload + .py() + .import("json")? + .call_method("dumps", (payload,), Some(&options))? + .extract(); + }; + dumps( + &value, + &JsonDialect::Stdlib(StdlibJsonOptions::new(None).with_separators(",", ":")), + ) + .map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + +/// The label of one test or result: its name, plus source and values for parameterized cases. +fn test_label(item: &Bound<'_, PyAny>, name_field: &str) -> PyResult { + let name: String = item.getattr(name_field)?.extract()?; + let source_path: Bound<'_, PyAny> = item.getattr("source_path")?; + let values: Vec> = item + .getattr("parameter_values")? + .try_iter()? + .collect::>()?; + if source_path.is_none() || values.is_empty() { + return Ok(name); + } + let mut schema: HashMap> = HashMap::new(); + for declaration in item.getattr("parameter_schema")?.try_iter()? { + let declaration: Bound<'_, PyAny> = declaration?; + schema.insert(declaration.getattr("name")?.extract()?, declaration); + } + let mut parameters: Vec = Vec::new(); + for pair in values { + let (value_name, value): (String, Bound<'_, PyAny>) = pair.extract()?; + let declaration: &Bound<'_, PyAny> = schema + .get(&value_name) + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(value_name.clone()))?; + let raw: Bound<'_, PyAny> = value.getattr("value")?; + let kind: String = value.getattr("kind")?.getattr("value")?.extract()?; + let payload: Bound<'_, PyAny> = if kind == DECIMAL_KIND { + raw.str()?.into_any() + } else { + raw + }; + parameters.push(TestParameter { + name: value_name, + value_type: declaration + .getattr("value_type")? + .getattr("value")? + .extract()?, + nullable: declaration.getattr("nullable")?.is_truthy()?, + rendered: compact_json(&payload)?, + }); + } + let path: String = source_path.call_method0("as_posix")?.extract()?; + Ok(parameterized_test_label(&name, Some(&path), ¶meters)) +} + +/// The shared name column width of test rows and their expectation rows. +fn name_width(entries: &Bound<'_, PyAny>) -> PyResult { + let mut labels: Vec = Vec::new(); + let mut children: Vec = Vec::new(); + for entry in entries.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + labels.push(test_label(&entry, "name")?); + for step in entry.getattr("chain")?.try_iter()? { + let step: Bound<'_, PyAny> = step?; + if step.getattr("expected_cte_sql")?.is_truthy()? { + children.push(format!( + "expected {}", + step.getattr("model_name")?.extract::()? + )); + } + } + for assertion in entry.getattr("assertions")?.try_iter()? { + children.push(format!( + "assertion {}", + assertion?.getattr("name")?.extract::()? + )); + } + } + Ok(test_name_width(&labels, &children)) +} + +fn outcome_status(outcome: &Bound<'_, PyAny>) -> PyResult<&'static str> { + Ok(test_outcome_status( + &outcome.getattr("value")?.extract::()?, + )) +} + +/// Nested `expect` rows of a completed test, one per step. +fn expectation_rows<'py>(result: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = result.py(); + let row_class: Bound<'py, PyAny> = imported(py, PROGRESS_MODELS, "NestedProgressChildRow")?; + let name: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_name")?; + let detail: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_detail")?; + let mut rows: Vec> = Vec::new(); + for step in result.getattr("step_results")?.try_iter()? { + let step: Bound<'py, PyAny> = step?; + rows.push(call_with( + &row_class, + &[ + ("label", text(py, EXPECT_LABEL)), + ("name", name.call1((step.getattr("model_name")?,))?), + ( + "status_text", + text(py, outcome_status(&step.getattr("outcome")?)?), + ), + ("detail", detail.call1((&step,))?), + ], + )?); + } + PyTuple::new(py, rows) +} + +/// The model a planned test's progress rows are grouped under. +fn entry_group(entry: &Bound<'_, PyAny>) -> PyResult { + let mut chain: Vec<(String, bool)> = Vec::new(); + for step in entry.getattr("chain")?.try_iter()? { + let step: Bound<'_, PyAny> = step?; + chain.push(( + step.getattr("model_name")?.extract()?, + !step.getattr("expected_cte_sql")?.is_none(), + )); + } + Ok(test_group_name(&chain)) +} + +/// Forward test progress, completions and connection timing to their reporters. +#[pyclass(module = "sqlbuild._native", frozen)] +struct TestCallbackRelay { + progress: Py, + event_writer: Py, + plan_output: Py, + connection: Py, +} + +#[pymethods] +impl TestCallbackRelay { + fn on_test_start(&self, py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult<()> { + call_with( + &self.progress.bind(py).getattr("on_item_start")?, + &[ + ("group_name", text(py, &entry_group(entry)?)), + ("item_name", text(py, &test_label(entry, "name")?)), + ("canonical_resource_name", entry.getattr("name")?), + ], + )?; + Ok(()) + } + + fn on_test_complete(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let mut group: String = String::new(); + if let Some(step) = result.getattr("step_results")?.try_iter()?.next() { + group = step?.getattr("model_name")?.extract()?; + } + if group.is_empty() { + group = UNKNOWN_GROUP.to_owned(); + } + let resource_id: Bound<'_, PyAny> = call_with( + &imported(py, RESOURCE_ID_MODULE, "sql_test_resource_id")?, + &[ + ("test_name", result.getattr("test_name")?), + ("source_path", result.getattr("source_path")?), + ("block_index", result.getattr("block_index")?), + ("case_name", result.getattr("case_name")?), + ], + )?; + call_with( + &self.progress.bind(py).getattr("on_item_complete")?, + &[ + ("group_name", text(py, &group)), + ("item_name", text(py, &test_label(result, "test_name")?)), + ( + "status_text", + text(py, outcome_status(&result.getattr("outcome")?)?), + ), + ("child_rows", expectation_rows(result)?.into_any()), + ("error_code", result.getattr("error_code")?), + ("error_help", result.getattr("error_help")?), + ("error_message", result.getattr("error_message")?), + ("canonical_resource_name", result.getattr("test_name")?), + ("canonical_resource_id", resource_id), + ], + )?; + call_with( + &self.event_writer.bind(py).getattr("write_build_result")?, + &[ + ("result", result.clone()), + ("plan", self.plan_output.bind(py).clone()), + ("command", text(py, "test")), + ], + )?; + Ok(()) + } + + fn on_connection_complete( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.connection_call( + py, + "on_connection_complete", + (connection_count, elapsed_seconds), + ) + } + + fn on_connection_error( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.connection_call( + py, + "on_connection_error", + (connection_count, elapsed_seconds), + ) + } +} + +impl TestCallbackRelay { + fn connection_call( + &self, + py: Python<'_>, + method: &str, + (connection_count, elapsed_seconds): (i64, f64), + ) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("connection_count", connection_count)?; + options.set_item("elapsed_seconds", elapsed_seconds)?; + self.connection + .bind(py) + .call_method(method, (), Some(&options))?; + Ok(()) + } +} + +fn unknown_case(py: Python<'_>, message: String) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", UNKNOWN_CASE_CODE)?; + imported(py, PLANNER_ERRORS, "PlannerInputError")?.call((message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Compile the selected tests and refuse a `--case` that matched none of them. +fn compile_tests<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; + let adapter: Bound<'py, PyAny> = invocation.getattr("adapter")?; + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.getattr("effective_project_dir")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let options: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &[ + ("selected_target", request.getattr("selected_target")?), + ("no_sql_validation", request.getattr("no_sql_validation")?), + ( + "source_deferral_enabled", + PyBool::new(py, false).to_owned().into_any(), + ), + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ( + "accepts_unit_test_selectors", + PyBool::new(py, true).to_owned().into_any(), + ), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("cli_vars", request.getattr("cli_vars")?), + ("external_sql_reference_resolver", resolver), + ], + )?; + let context: Bound<'py, PyAny> = call_with( + &imported(py, STATIC_COMMAND_MODULE, "compile_static_command_context")?, + &[ + ("discovered_inputs", discovered), + ("adapter", adapter.clone()), + ("options", options), + ( + "on_progress", + invocation + .getattr("planning_progress")? + .getattr("on_progress")?, + ), + ], + )?; + let case_name: Bound<'py, PyAny> = request.getattr("case_name")?; + let plan_output: Bound<'py, PyAny> = call_with( + &imported(py, TEST_PLAN_MODULE, "build_test_command_plan")?, + &[ + ("project", context.getattr("project")?), + ("adapter", adapter), + ("scope", context.getattr("scope")?), + ("relations", context.getattr("relations")?), + ("case_name", case_name.clone()), + ], + )?; + if !case_name.is_none() && !plan_output.getattr("test_entries")?.is_truthy()? { + let cases: Vec = strings(plan_output.getattr("available_test_case_names")?)?; + let available: String = if cases.is_empty() { + "none".to_owned() + } else { + cases.join(", ") + }; + return Err(unknown_case( + py, + format!( + "SQL test case '{}' did not match the selected tests; available cases: {available}", + case_name.str()? + ), + )); + } + call_with( + &imported(py, NATIVE_MODULE, "build_static_pipeline_result")?, + &[("context", context), ("plan_output", plan_output)], + ) +} + +fn inspection_entry(entry: &Bound<'_, PyAny>) -> PyResult { + let case: Bound<'_, PyAny> = entry.getattr("case_name")?; + let name: String = entry.getattr("name")?.extract()?; + let label: String = if case.is_none() { + name + } else { + let parent: Bound<'_, PyAny> = entry.getattr("parent_name")?; + let parent: String = if parent.is_truthy()? { + parent.extract()? + } else { + name + }; + format!("{parent} [{}]", case.str()?) + }; + let mut real_models: Vec = Vec::new(); + let mut expected_models: Vec = Vec::new(); + for step in entry.getattr("chain")?.try_iter()? { + let step: Bound<'_, PyAny> = step?; + let model: String = step.getattr("model_name")?.extract()?; + if !step.getattr("expected_cte_sql")?.is_none() { + expected_models.push(model.clone()); + } + real_models.push(model); + } + Ok(TestInspectionEntry { + label, + mocked_sources: strings(entry.getattr("mock_source_names")?)?, + mocked_refs: strings(entry.getattr("mock_ref_names")?)?, + mocked_seeds: strings(entry.getattr("mock_seed_names")?)?, + mocked_dbt_refs: strings(entry.getattr("mock_dbt_ref_names")?)?, + mocked_table_functions: strings(entry.getattr("mock_table_function_names")?)?, + real_models, + expected_models, + }) +} + +/// Write the resolved plan without executing it; returns the exit code. +fn inspect(stream: &Bound<'_, PyAny>, plan_output: &Bound<'_, PyAny>) -> PyResult { + let entries: Vec = plan_output + .getattr("test_entries")? + .try_iter()? + .map(|entry| inspection_entry(&entry?)) + .collect::>()?; + let warnings: Vec = plan_output + .getattr("warnings")? + .try_iter()? + .map(|warning| { + let warning: Bound<'_, PyAny> = warning?; + Ok(TestPlanWarning { + severity: warning.getattr("severity")?.getattr("value")?.extract()?, + message: warning.getattr("message")?.str()?.extract()?, + }) + }) + .collect::>()?; + let (rendered, code) = render_test_inspection(&entries, &warnings); + stream.call_method1("write", (rendered,))?; + stream.call_method0("flush")?; + Ok(code) +} + +/// Write the ready header and run the tests with nested progress; returns the results. +fn run_tests<'py>( + invocation: &Bound<'py, PyAny>, + pipeline: &Bound<'py, PyAny>, + concurrency: i64, +) -> PyResult>> { + let py: Python<'py> = invocation.py(); + let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; + let use_color: Bound<'py, PyAny> = invocation.getattr("use_color")?; + let style: CliStyle = CliStyle { + use_color: use_color.is_truthy()?, + }; + let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let entries: Bound<'py, PyAny> = plan_output.getattr("test_entries")?; + let test_count: usize = entries.len()?; + let mut models: BTreeSet = BTreeSet::new(); + for entry in entries.try_iter()? { + for step in entry?.getattr("chain")?.try_iter()? { + models.insert(step?.getattr("model_name")?.extract()?); + } + } + let worker_count: i64 = concurrency.min(i64::try_from(test_count).unwrap_or(i64::MAX)); + let progress: Bound<'py, PyAny> = call_with( + &imported(py, NESTED_PROGRESS_MODULE, "NestedCommandProgressCallbacks")?, + &[ + ("total", PyInt::new(py, test_count).into_any()), + ("label", text(py, "test")), + ("stream", stream.clone()), + ("use_color", use_color.clone()), + ( + "name_width", + PyInt::new(py, name_width(&entries)?).into_any(), + ), + ("concurrency", PyInt::new(py, worker_count).into_any()), + ], + )?; + let projector: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + if !projector.is_none() { + for entry in entries.try_iter()? { + call_with( + &projector.getattr("expect_resource_enrichment")?, + &[("resource_name", entry?.getattr("name")?)], + )?; + } + } + stream.call_method1( + "write", + (render_test_ready((test_count, models.len()), style),), + )?; + stream.call_method0("flush")?; + let connection: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", invocation.getattr("adapter_name")?), + ("stream", stream.clone()), + ("use_color", use_color.clone()), + ], + )?; + let preflight: Bound<'py, PyAny> = call_with( + &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, + &[("stream", stream), ("use_color", use_color)], + )?; + let event_writer: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; + let relay: Bound<'py, TestCallbackRelay> = Bound::new( + py, + TestCallbackRelay { + progress: progress.unbind(), + event_writer: event_writer.clone().unbind(), + plan_output: plan_output.clone().unbind(), + connection: connection.clone().unbind(), + }, + )?; + let executed: PyResult>> = (|| { + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTOR_PIPELINE_MODELS, "TestPipelineCallbacks")?, + &[ + ( + "on_connection_start", + connection.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + relay.getattr("on_connection_complete")?, + ), + ("on_connection_error", relay.getattr("on_connection_error")?), + ( + "on_progress", + preflight.getattr("report_preflight_progress")?, + ), + ("on_test_start", relay.getattr("on_test_start")?), + ("on_test_complete", relay.getattr("on_test_complete")?), + ], + )?; + call_with( + &imported(py, EXECUTOR_RUN_MODULE, "run_test_pipeline")?, + &[ + ("plan", plan_output), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("adapter", invocation.getattr("adapter")?), + ("max_concurrency", PyInt::new(py, concurrency).into_any()), + ("callbacks", callbacks), + ("run_id", pipeline.getattr("project")?.getattr("run_id")?), + ], + )? + .try_iter()? + .collect() + })(); + let closed: PyResult> = event_writer.call_method0("close"); + let results: Vec> = executed?; + closed?; + Ok(results) +} + +/// Write the summary, failures and JSON output; returns the exit code. +fn complete( + request: &Bound<'_, PyAny>, + invocation: &Bound<'_, PyAny>, + results: &[Bound<'_, PyAny>], +) -> PyResult { + let py: Python<'_> = request.py(); + let mut failures: Vec = Vec::new(); + for result in results { + if result + .getattr("outcome")? + .getattr("value")? + .extract::()? + == PASS_OUTCOME + { + continue; + } + failures.push(TestFailure { + label: test_label(result, "test_name")?, + error_message: result.getattr("error_message")?.extract()?, + error_code: result.getattr("error_code")?.extract()?, + error_help: result.getattr("error_help")?.extract()?, + }); + } + let passed: usize = results.len() - failures.len(); + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + let style: CliStyle = CliStyle { + use_color: invocation.getattr("use_color")?.is_truthy()?, + }; + stream.call_method1( + "write", + (render_test_completion( + (passed, results.len()), + &failures, + style, + ),), + )?; + stream.call_method0("flush")?; + let tuple: Bound<'_, PyAny> = PyTuple::new(py, results)?.into_any(); + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_test_execution_json")?, + &[("results", tuple)], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(test_exit_code(passed, results.len())) +} + +/// Run one `sqb test` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let inspect_only: bool = request.getattr("inspect")?.is_truthy()?; + let invocation: Bound<'_, PyAny> = call_with( + &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, + &[ + ("request", request.clone()), + ( + "invocation_type", + imported(py, COMMAND_MODELS, "TestInvocation")?, + ), + ( + "require_connection", + PyBool::new(py, !inspect_only).to_owned().into_any(), + ), + ], + )?; + let pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + if inspect_only { + return inspect(&stream, &pipeline.getattr("plan_output")?); + } + let requested: Bound<'_, PyAny> = request.getattr("concurrency")?; + let concurrency: i64 = if requested.is_none() { + pipeline + .getattr("project")? + .getattr("settings")? + .getattr("concurrency")? + .extract()? + } else { + requested.extract()? + }; + stream.call_method1("write", ("\n",))?; + let header: Bound<'_, PyDict> = PyDict::new(py); + header.set_item("stream", &stream)?; + header.set_item("command", EXECUTION_COMMAND)?; + header.set_item("target", py.None())?; + header.set_item("concurrency", concurrency)?; + header.set_item("use_color", invocation.getattr("use_color")?)?; + imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&header))?; + let results: Vec> = run_tests(&invocation, &pipeline, concurrency)?; + call_with( + &imported(py, RUNTIME_TARGET_MODULE, "write_test_runtime_target")?, + &[ + ( + "target_dir", + invocation + .getattr("effective_project_dir")? + .call_method1("__truediv__", ("target",))?, + ), + ("adapter", invocation.getattr("adapter")?), + ("plan_output", pipeline.getattr("plan_output")?), + ("results", PyTuple::new(py, &results)?.into_any()), + ], + )?; + complete(request, &invocation, &results) +} + +/// Add source and safe typed values only for parameterized test rows. +#[pyfunction] +#[pyo3(signature = (**options))] +fn format_parameterized_test_label( + py: Python<'_>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "format_parameterized_test_label", + options, + ( + &[ + "name", + "source_path", + "parameter_schema", + "parameter_values", + ], + &[], + ), + )?; + let item: Bound<'_, PyAny> = py.import("types")?.getattr("SimpleNamespace")?.call( + (), + Some(&{ + let fields: Bound<'_, PyDict> = PyDict::new(py); + for field in [ + "name", + "source_path", + "parameter_schema", + "parameter_values", + ] { + fields.set_item(field, arguments.value(field)?)?; + } + fields + }), + )?; + test_label(&item, "name") +} + +/// Resolve one shared width for test rows and nested expectation rows. +#[pyfunction] +fn resolve_test_name_width(test_entries: &Bound<'_, PyAny>) -> PyResult { + name_width(test_entries) +} + +/// Build nested expectation rows for a completed SQL unit test. +#[pyfunction] +fn build_test_expectation_rows<'py>(result: &Bound<'py, PyAny>) -> PyResult> { + expectation_rows(result) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_test_command, module)?)?; + module.add_function(wrap_pyfunction!(format_parameterized_test_label, module)?)?; + module.add_function(wrap_pyfunction!(resolve_test_name_width, module)?)?; + module.add_function(wrap_pyfunction!(build_test_expectation_rows, module)?)?; + Ok(()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 15ffe53341..dac200dbd3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -29,6 +29,7 @@ from sqlbuild.cli.commands.models import ( PlaygroundCommandRequest, RefactorCommandRequest, ScopeCommandRequest, + TestCommandRequest, ) from sqlbuild.cli.commands.types import CompileLineageMode, ScopeIndexLoader from sqlbuild.cli.compile.models import CompileAnalysis @@ -40,6 +41,7 @@ from sqlbuild.cli.entry.models import ( ResolvedWarehouse, ) from sqlbuild.cli.output.models import SkillMaintenanceResult, SkillUpdateResult, WrittenTarget +from sqlbuild.cli.progress.models import NestedProgressChildRow from sqlbuild.compiler.auditing.models import ( AuditGateIdentity, MeasurementContract, @@ -133,6 +135,7 @@ from sqlbuild.compiler.discovery.models import ( DiscoveryFileFault, EnumDeclaration, ModelSchemaDeclaration, + SqlTestParameterDeclaration, TolerantScopeDiscovery, ) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet @@ -162,6 +165,7 @@ from sqlbuild.compiler.planner.models import ( FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, PlanOutput, + SqlTestPlanEntry, ) from sqlbuild.compiler.planner.types import CursorGrain from sqlbuild.compiler.python_nodes.models import ( @@ -2889,6 +2893,18 @@ def write_check_results( python_graph: PythonNodeGraph | None = None, ) -> None: ... def format_check_json(*, results: tuple[PythonCheckExecutionResult, ...]) -> str: ... +def run_test_command(*, request: TestCommandRequest) -> int: ... +def format_parameterized_test_label( + *, + name: str, + source_path: Path | None, + parameter_schema: tuple[SqlTestParameterDeclaration, ...], + parameter_values: tuple[tuple[str, SqlValue], ...], +) -> str: ... +def resolve_test_name_width(test_entries: tuple[SqlTestPlanEntry, ...]) -> int: ... +def build_test_expectation_rows( + result: SqlTestExecutionResult, +) -> tuple[NestedProgressChildRow, ...]: ... def select_load_entries( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/cli/commands/_helpers/test/execution.py b/src/sqlbuild/cli/commands/_helpers/test/execution.py deleted file mode 100644 index 8e5d31fd76..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/execution.py +++ /dev/null @@ -1,208 +0,0 @@ -"""Test command execution phases.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import ( - ConnectionProgressReporter, - ExecutionEventWriter, - NativeProgressProjector, - current_native_progress_projector, -) -from sqlbuild.cli.commands._helpers.test.sql_progress import ( - build_test_expectation_rows, - format_parameterized_test_label, - resolve_test_name_width, - test_outcome_status, -) -from sqlbuild.cli.commands.models import ( - TestCommandRequest, - TestExecutionPreparation, - TestInvocation, -) -from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( - NestedCommandProgressCallbacks, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.executor.pipeline.main.run import run_test_pipeline -from sqlbuild.executor.pipeline.models import TestPipelineCallbacks -from sqlbuild.executor.testing.main.resource_id import sql_test_resource_id -from sqlbuild.executor.testing.models import SqlTestExecutionResult -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.surface_header import format_surface_header - - -def prepare_test_execution( - *, - request: TestCommandRequest, - invocation: TestInvocation, - pipeline_result: CompilePipelineResult, -) -> TestExecutionPreparation: - """Prepare nested progress reporting and write the test section header.""" - - test_count: int = len(pipeline_result.plan_output.test_entries) - effective_concurrency: int = resolve_test_concurrency( - request=request, - pipeline_result=pipeline_result, - ) - worker_count: int = min(effective_concurrency, test_count) - model_names: set[str] = set() - for entry in pipeline_result.plan_output.test_entries: - for step in entry.chain: - model_names.add(step.model_name) - model_count: int = len(model_names) - style: CliStyle = CliStyle(use_color=invocation.use_color) - styled_header: str = format_surface_header( - style=style, - title="Test ready", - context=f"{test_count} selected, {model_count} models", - ) - progress: NestedCommandProgressCallbacks = NestedCommandProgressCallbacks( - total=test_count, - label="test", - stream=invocation.progress_stream, - use_color=invocation.use_color, - name_width=resolve_test_name_width(pipeline_result.plan_output.test_entries), - concurrency=worker_count, - ) - projector: NativeProgressProjector | None = current_native_progress_projector() - if projector is not None: - for entry in pipeline_result.plan_output.test_entries: - projector.expect_resource_enrichment(resource_name=entry.name) - invocation.progress_stream.write(f"\n{styled_header}\n\n") - invocation.progress_stream.flush() - execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=invocation.adapter_name, - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - preflight_progress: TransientStatusReporter = TransientStatusReporter( - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - return TestExecutionPreparation( - progress=progress, - execution_connection_progress=execution_connection_progress, - preflight_progress=preflight_progress, - effective_concurrency=effective_concurrency, - worker_count=worker_count, - ) - - -def resolve_test_concurrency( - *, - request: TestCommandRequest, - pipeline_result: CompilePipelineResult, -) -> int: - """Resolve CLI override before the project-wide concurrency setting.""" - - return ( - request.concurrency - if request.concurrency is not None - else pipeline_result.project.settings.concurrency - ) - - -def execute_test_plan( - *, - invocation: TestInvocation, - pipeline_result: CompilePipelineResult, - preparation: TestExecutionPreparation, -) -> tuple[SqlTestExecutionResult, ...]: - """Execute the test plan with nested progress reporting.""" - - event_writer: ExecutionEventWriter = ExecutionEventWriter() - on_complete: Callable[[SqlTestExecutionResult], None] = _build_on_complete( - progress=preparation.progress, - event_writer=event_writer, - pipeline_result=pipeline_result, - ) - try: - return run_test_pipeline( - plan=pipeline_result.plan_output, - connection_config=invocation.connection_config, - adapter=invocation.adapter, - max_concurrency=preparation.effective_concurrency, - callbacks=TestPipelineCallbacks( - on_connection_start=(preparation.execution_connection_progress.on_connection_start), - on_connection_complete=lambda connection_count, elapsed_seconds: ( - preparation.execution_connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - preparation.execution_connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_progress=preparation.preflight_progress.report_preflight_progress, - on_test_start=lambda entry: preparation.progress.on_item_start( - group_name=_test_group_name_from_entry(entry), - item_name=format_parameterized_test_label( - name=entry.name, - source_path=entry.source_path, - parameter_schema=entry.parameter_schema, - parameter_values=entry.parameter_values, - ), - canonical_resource_name=entry.name, - ), - on_test_complete=on_complete, - ), - run_id=pipeline_result.project.run_id, - ) - finally: - event_writer.close() - - -def _build_on_complete( - *, - progress: NestedCommandProgressCallbacks, - event_writer: ExecutionEventWriter, - pipeline_result: CompilePipelineResult, -) -> Callable[[SqlTestExecutionResult], None]: - def _on_complete(result: SqlTestExecutionResult) -> None: - model_name: str = "" - if result.step_results: - model_name = result.step_results[0].model_name - group_name: str = model_name or "(unknown)" - status_text: str = test_outcome_status(outcome=result.outcome) - progress.on_item_complete( - group_name=group_name, - item_name=format_parameterized_test_label( - name=result.test_name, - source_path=result.source_path, - parameter_schema=result.parameter_schema, - parameter_values=result.parameter_values, - ), - status_text=status_text, - child_rows=build_test_expectation_rows(result), - error_code=result.error_code, - error_help=result.error_help, - error_message=result.error_message, - canonical_resource_name=result.test_name, - canonical_resource_id=sql_test_resource_id( - test_name=result.test_name, - source_path=result.source_path, - block_index=result.block_index, - case_name=result.case_name, - ), - ) - event_writer.write_build_result( - result=result, - plan=pipeline_result.plan_output, - command="test", - ) - - return _on_complete - - -def _test_group_name_from_entry(entry: SqlTestPlanEntry) -> str: - for step in entry.chain: - if step.expected_cte_sql is not None: - return step.model_name - if entry.chain: - return entry.chain[0].model_name - return "(unknown)" diff --git a/src/sqlbuild/cli/commands/_helpers/test/inspection.py b/src/sqlbuild/cli/commands/_helpers/test/inspection.py deleted file mode 100644 index e518269e4a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/inspection.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Resolved SQL-test plan inspection output.""" - -from __future__ import annotations - -from collections.abc import Iterable -from typing import TextIO - -from sqlbuild.compiler.planner.models import PlanOutput, PlanWarning, SqlTestPlanEntry -from sqlbuild.compiler.planner.types import WarningSeverity - - -def write_test_plan_inspection(*, stream: TextIO, plan_output: PlanOutput) -> int: - """Write selected test fixture boundaries and real model chains without executing them.""" - - stream.write("\nResolved test plan\n\n") - for entry in plan_output.test_entries: - stream.write(f"{_entry_label(entry)}\n") - _write_names(stream=stream, label="mocked sources", names=entry.mock_source_names) - _write_names(stream=stream, label="mocked refs", names=entry.mock_ref_names) - _write_names(stream=stream, label="mocked seeds", names=entry.mock_seed_names) - _write_names(stream=stream, label="mocked dbt refs", names=entry.mock_dbt_ref_names) - _write_names( - stream=stream, - label="mocked table functions", - names=entry.mock_table_function_names, - ) - _write_names( - stream=stream, - label="real models", - names=tuple(step.model_name for step in entry.chain), - ) - _write_names( - stream=stream, - label="expected models", - names=tuple( - step.model_name for step in entry.chain if step.expected_cte_sql is not None - ), - ) - for mock_name in entry.mock_ref_names: - stream.write(f" boundary: {mock_name} is replaced by __ref__{mock_name}\n") - stream.write("\n") - error_warnings: tuple[PlanWarning, ...] = tuple( - warning for warning in plan_output.warnings if warning.severity == WarningSeverity.ERROR - ) - for warning in plan_output.warnings: - stream.write(f" {warning.severity.value}: {warning.message}\n") - outcome: str = "failed" if error_warnings else "complete" - stream.write( - f"Test plan inspection {outcome}: {len(plan_output.test_entries)} selected, " - f"{len(error_warnings)} errors.\n" - ) - stream.flush() - return 1 if error_warnings else 0 - - -def _write_names(*, stream: TextIO, label: str, names: Iterable[str]) -> None: - values: tuple[str, ...] = tuple(names) - stream.write(f" {label}: {', '.join(values) if values else 'none'}\n") - - -def _entry_label(entry: SqlTestPlanEntry) -> str: - if entry.case_name is None: - return entry.name - return f"{entry.parent_name or entry.name} [{entry.case_name}]" diff --git a/src/sqlbuild/cli/commands/_helpers/test/invocation.py b/src/sqlbuild/cli/commands/_helpers/test/invocation.py deleted file mode 100644 index 562d137d69..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/invocation.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Test command invocation resolution phase.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( - resolve_reported_command_invocation, -) -from sqlbuild.cli.commands.models import TestCommandRequest, TestInvocation - - -def resolve_test_invocation(*, request: TestCommandRequest) -> TestInvocation: - """Resolve test context; ``--inspect`` never connects, so it needs no connection.""" - - return resolve_reported_command_invocation( - request=request, - invocation_type=TestInvocation, - require_connection=not request.inspect, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/test/outputs.py b/src/sqlbuild/cli/commands/_helpers/test/outputs.py deleted file mode 100644 index dca9332ef6..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/outputs.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Test command output writing and exit-code phases.""" - -from __future__ import annotations - -from sqlbuild._native import ( - format_summary_footer, - format_test_execution_json, - write_execution_json_output, -) -from sqlbuild.cli.commands._helpers.test.sql_progress import format_parameterized_test_label -from sqlbuild.cli.commands.models import TestCommandRequest, TestInvocation -from sqlbuild.executor.testing.models import SqlTestExecutionResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.coded_error_text import format_coded_error - -_ERROR_LABEL_WIDTH: int = 10 - - -def write_test_completion_output( - *, - request: TestCommandRequest, - invocation: TestInvocation, - results: tuple[SqlTestExecutionResult, ...], -) -> None: - """Write the test summary footer and optional JSON output.""" - - pass_count: int = sum(1 for r in results if r.outcome == SqlTestOutcome.PASS) - fail_count: int = len(results) - pass_count - summary: str = format_summary_footer( - counts=( - ("PASS", pass_count), - ("FAIL", fail_count), - ("TOTAL", len(results)), - ), - use_color=invocation.use_color, - ) - failure_details: str = _format_test_failure_details( - results=results, - use_color=invocation.use_color, - ) - rendered: str = f"\n{summary}" - if failure_details: - rendered = f"{rendered}\n\n{failure_details}" - invocation.progress_stream.write(f"{rendered}\n") - invocation.progress_stream.flush() - write_execution_json_output( - payload=format_test_execution_json(results=results), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def resolve_test_exit_code(results: tuple[SqlTestExecutionResult, ...]) -> int: - """Resolve the test exit code from failed test outcomes.""" - - pass_count: int = sum(1 for r in results if r.outcome == SqlTestOutcome.PASS) - return 0 if len(results) - pass_count == 0 else 1 - - -def _format_test_failure_details( - *, results: tuple[SqlTestExecutionResult, ...], use_color: bool -) -> str: - failed_results: tuple[SqlTestExecutionResult, ...] = tuple( - result for result in results if result.outcome != SqlTestOutcome.PASS - ) - if not failed_results: - return "" - style: CliStyle = CliStyle(use_color=use_color) - lines: list[str] = [style.error_strong("Failures:"), ""] - result: SqlTestExecutionResult - for result in failed_results: - lines.append( - " " - + format_parameterized_test_label( - name=result.test_name, - source_path=result.source_path, - parameter_schema=result.parameter_schema, - parameter_values=result.parameter_values, - ) - ) - error_message: str = result.error_message or "test failed" - rendered_error: str = error_message - if result.error_code is not None: - rendered_error = format_coded_error( - code=result.error_code, - message=error_message, - help=result.error_help, - use_color=use_color, - include_error_label=False, - ) - label: str = style.error_muted(f"{'error':<{_ERROR_LABEL_WIDTH}}") - error_line: str - for index, error_line in enumerate(rendered_error.splitlines() or [rendered_error]): - display_label: str = label if index == 0 else " " * _ERROR_LABEL_WIDTH - lines.append(f" {display_label}{error_line}") - lines.append("") - return "\n".join(lines).rstrip() diff --git a/src/sqlbuild/cli/commands/_helpers/test/planning.py b/src/sqlbuild/cli/commands/_helpers/test/planning.py deleted file mode 100644 index 7be9b302ba..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/planning.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Test command plan compilation phase.""" - -from __future__ import annotations - -from sqlbuild._native import build_static_pipeline_result -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.models import TestCommandRequest, TestInvocation -from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, - StaticCommandContext, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.commands.sql_test import build_test_command_plan -from sqlbuild.compiler.planner.models import PlanOutput - - -def compile_test_plan( - *, - request: TestCommandRequest, - invocation: TestInvocation, -) -> CompilePipelineResult: - """Compile the test plan for the selected scope.""" - - options: CompilePipelineOptions = CompilePipelineOptions( - selected_target=request.selected_target, - no_sql_validation=request.no_sql_validation, - source_deferral_enabled=False, - select=request.select, - exclude=request.exclude, - accepts_unit_test_selectors=True, - connection_config=invocation.connection_config, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ) - context: StaticCommandContext = compile_static_command_context( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - options=options, - on_progress=invocation.planning_progress.on_progress, - ) - plan_output: PlanOutput = build_test_command_plan( - project=context.project, - adapter=invocation.adapter, - scope=context.scope, - relations=context.relations, - case_name=request.case_name, - ) - if request.case_name is not None: - plan_output = _validate_parameter_case_selection( - plan_output=plan_output, - case_name=request.case_name, - available_cases=plan_output.available_test_case_names, - ) - return build_static_pipeline_result( - context=context, - plan_output=plan_output, - ) - - -def _validate_parameter_case_selection( - *, plan_output: PlanOutput, case_name: str, available_cases: tuple[str, ...] -) -> PlanOutput: - if plan_output.test_entries: - return plan_output - available_text: str = ", ".join(available_cases) or "none" - raise PlannerInputError( - f"SQL test case '{case_name}' did not match the selected tests; available cases: " - f"{available_text}", - code="S008", - ) diff --git a/src/sqlbuild/cli/commands/_helpers/test/sql_progress.py b/src/sqlbuild/cli/commands/_helpers/test/sql_progress.py deleted file mode 100644 index ffce24acfb..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/sql_progress.py +++ /dev/null @@ -1,103 +0,0 @@ -"""SQL unit-test progress formatting helpers.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from sqlbuild._native import ( - format_expectation_detail, - format_expectation_name, - resolve_name_column_width, -) -from sqlbuild.cli.progress.constants import NESTED_CHILD_NAME_INDENT -from sqlbuild.cli.progress.models import NestedProgressChildRow -from sqlbuild.compiler.discovery.models import SqlTestParameterDeclaration -from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind - - -def resolve_test_name_width(test_entries: tuple[SqlTestPlanEntry, ...]) -> int: - """Resolve one shared width for test rows and nested expectation rows.""" - - names: list[str] = [] - child_names: list[str] = [] - entry: SqlTestPlanEntry - for entry in test_entries: - names.append( - format_parameterized_test_label( - name=entry.name, - source_path=entry.source_path, - parameter_schema=entry.parameter_schema, - parameter_values=entry.parameter_values, - ) - ) - child_names.extend( - f"expected {step.model_name}" for step in entry.chain if step.expected_cte_sql - ) - child_names.extend(f"assertion {assertion.name}" for assertion in entry.assertions) - child_name_length: int = max((len(name) for name in child_names), default=0) - return max( - resolve_name_column_width(names=names, min_width=50), - child_name_length + NESTED_CHILD_NAME_INDENT, - ) - - -def build_test_expectation_rows( - result: SqlTestExecutionResult, -) -> tuple[NestedProgressChildRow, ...]: - """Build nested expectation rows for a completed SQL unit test.""" - - rows: list[NestedProgressChildRow] = [] - step_result: StepResult - for step_result in result.step_results: - expectation_name: str = format_expectation_name(step_result.model_name) - status_text: str = test_outcome_status(outcome=step_result.outcome) - rows.append( - NestedProgressChildRow( - label="expect", - name=expectation_name, - status_text=status_text, - detail=format_expectation_detail(step_result), - ) - ) - return tuple(rows) - - -def format_parameterized_test_label( - *, - name: str, - source_path: Path | None, - parameter_schema: tuple[SqlTestParameterDeclaration, ...], - parameter_values: tuple[tuple[str, SqlValue], ...], -) -> str: - """Add source and safe typed values only for parameterized test rows.""" - - if source_path is None or not parameter_values: - return name - schema_by_name: dict[str, SqlTestParameterDeclaration] = { - parameter.name: parameter for parameter in parameter_schema - } - rendered_values: list[str] = [] - value_name: str - value: SqlValue - for value_name, value in parameter_values: - declaration: SqlTestParameterDeclaration = schema_by_name[value_name] - payload: object = str(value.value) if value.kind == SqlValueKind.DECIMAL else value.value - rendered: str = json.dumps(payload, ensure_ascii=True, separators=(",", ":")) - nullable: str = "?" if declaration.nullable else "" - rendered_values.append(f"{value_name}:{declaration.value_type.value}{nullable}={rendered}") - return f"{name} ({source_path.as_posix()}; {', '.join(rendered_values)})" - - -def test_outcome_status(*, outcome: SqlTestOutcome) -> str: - """Preserve the semantic distinction between assertion failures and execution errors.""" - - if outcome == SqlTestOutcome.PASS: - return "PASS" - if outcome == SqlTestOutcome.ERROR: - return "ERROR" - return "FAIL" diff --git a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py index eb5a92b134..8beb7512a2 100644 --- a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py +++ b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py @@ -16,12 +16,12 @@ format_completion_line, format_expectation_detail, format_expectation_name, + format_parameterized_test_label, format_status_cell, format_summary_footer, ) from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.commands._helpers.test.sql_progress import format_parameterized_test_label from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.types import CompiledResourceType diff --git a/src/sqlbuild/cli/commands/main/execution/_test.py b/src/sqlbuild/cli/commands/main/execution/_test.py deleted file mode 100644 index 52114df624..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_test.py +++ /dev/null @@ -1,70 +0,0 @@ -"""CLI test command entry point.""" - -from __future__ import annotations - -from sqlbuild._native import write_execution_header -from sqlbuild.cli.commands._helpers.test.execution import ( - execute_test_plan, - prepare_test_execution, - resolve_test_concurrency, -) -from sqlbuild.cli.commands._helpers.test.inspection import write_test_plan_inspection -from sqlbuild.cli.commands._helpers.test.invocation import resolve_test_invocation -from sqlbuild.cli.commands._helpers.test.outputs import ( - resolve_test_exit_code, - write_test_completion_output, -) -from sqlbuild.cli.commands._helpers.test.planning import compile_test_plan -from sqlbuild.cli.commands.models import ( - TestCommandRequest, - TestExecutionPreparation, - TestInvocation, -) -from sqlbuild.cli.target_artifacts.main._write_test_runtime_target import write_test_runtime_target -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.executor.testing.models import SqlTestExecutionResult - - -def run_test(request: TestCommandRequest) -> int: - """Execute the test command.""" - - invocation: TestInvocation = resolve_test_invocation(request=request) - pipeline_result: CompilePipelineResult = compile_test_plan( - request=request, - invocation=invocation, - ) - if request.inspect: - return write_test_plan_inspection( - stream=invocation.progress_stream, - plan_output=pipeline_result.plan_output, - ) - invocation.progress_stream.write("\n") - write_execution_header( - stream=invocation.progress_stream, - command="sqb test", - target=None, - concurrency=resolve_test_concurrency(request=request, pipeline_result=pipeline_result), - use_color=invocation.use_color, - ) - preparation: TestExecutionPreparation = prepare_test_execution( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - ) - results: tuple[SqlTestExecutionResult, ...] = execute_test_plan( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - ) - write_test_runtime_target( - target_dir=invocation.effective_project_dir / "target", - adapter=invocation.adapter, - plan_output=pipeline_result.plan_output, - results=results, - ) - write_test_completion_output( - request=request, - invocation=invocation, - results=results, - ) - return resolve_test_exit_code(results) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index ff1b64dfdd..4129431c76 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -49,9 +49,6 @@ DirectPythonLifecycleState, ) from sqlbuild.cli.progress.classes.audit_progress_reporter import AuditProgressReporter - from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( - NestedCommandProgressCallbacks, - ) from sqlbuild.compiler.compile.models import ( CompiledSqlScenario, CompilerDiagnostic, @@ -837,17 +834,6 @@ class TestInvocation: planning_progress: PlanningProgressReporter -@dataclass(frozen=True) -class TestExecutionPreparation: - """Prepared nested progress and execution reporters for test runs.""" - - progress: NestedCommandProgressCallbacks - execution_connection_progress: ConnectionProgressReporter - preflight_progress: TransientStatusReporter - effective_concurrency: int - worker_count: int - - @dataclass(frozen=True) class RulesCommandRequest: """Inputs for focused rule execution, inspection, or skill generation.""" diff --git a/src/sqlbuild/compiler/planner/main/commands/sql_test.py b/src/sqlbuild/compiler/planner/main/commands/_sql_test.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/commands/sql_test.py rename to src/sqlbuild/compiler/planner/main/commands/_sql_test.py diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py index e5883bc1d0..9f96cf02db 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py @@ -14,7 +14,6 @@ resolve_reported_command_invocation, ) from sqlbuild.cli.commands._helpers.seed.invocation import resolve_seed_invocation -from sqlbuild.cli.commands._helpers.test.invocation import resolve_test_invocation from tests.integration.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( CommandInvocationTestCase, ) @@ -50,7 +49,11 @@ ), CommandInvocationTestCase( description="test json output reports progress on stderr", - resolve=resolve_test_invocation, + resolve=partial( + resolve_reported_command_invocation, + invocation_type=command_models.TestInvocation, + require_connection=True, + ), request_type=command_models.TestCommandRequest, json_output=True, expected_invocation_type_name="TestInvocation", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index b1bbfd5dba..cddca28fcd 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -17,6 +17,7 @@ discover_project_inputs, format_load_execution_json, run_load_command, + run_test_command, select_load_entries, ) from sqlbuild.adapter.contract.types import CursorKind @@ -25,7 +26,6 @@ from sqlbuild.cli.commands.main.execution._audit import run_audit from sqlbuild.cli.commands.main.execution._build import run_build from sqlbuild.cli.commands.main.execution._scenario import run_scenario -from sqlbuild.cli.commands.main.execution._test import run_test from sqlbuild.cli.commands.main.project._plan import run_plan from sqlbuild.cli.commands.models import ( AuditCommandRequest, @@ -1232,7 +1232,7 @@ def test_given_no_managed_source_read_ambiguity_when_building_then_source_deferr "audit": lambda **kwargs: run_audit(AuditCommandRequest(**kwargs)), "build": lambda **kwargs: run_build(BuildCommandRequest(**kwargs)), "scenario": lambda **kwargs: run_scenario(ScenarioTestCommandRequest(**kwargs)), - "test": lambda **kwargs: run_test(TestCommandRequest(**kwargs)), + "test": lambda **kwargs: run_test_command(request=TestCommandRequest(**kwargs)), } exit_code: int = command_runners[test_case.command]( project_dir=tmp_path, diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 8b48016772..435f92a3c7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -9,13 +9,13 @@ import pytest -from sqlbuild._native import write_execution_header -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.commands._helpers.test.sql_progress import ( +from sqlbuild._native import ( build_test_expectation_rows, resolve_test_name_width, + write_execution_header, ) +from sqlbuild.adapter.contract.models import LifeCycleEvent +from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.cli.commands.classes import build_progress_callbacks from sqlbuild.cli.commands.classes.build_progress_callbacks import ( BuildProgressCallbacks, diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py index a9591a700e..84ae015d09 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py @@ -8,10 +8,9 @@ import pytest -from sqlbuild._native import format_audit_execution_json +from sqlbuild._native import format_audit_execution_json, format_parameterized_test_label from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.commands._helpers.test.sql_progress import format_parameterized_test_label from sqlbuild.compiler.auditing.models import ( MeasurementThresholdBound, MeasurementThresholds, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 318f7f9605..77a9ebd7ed 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -31,7 +31,7 @@ from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope -from sqlbuild.compiler.planner.main.commands.sql_test import build_test_command_plan +from sqlbuild.compiler.planner.main.commands._sql_test import build_test_command_plan from sqlbuild.compiler.planner.models import ( ChainStep, NativeSqlTestPlan, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 4faf51483d..8d032760ec 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -19,10 +19,10 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope +from sqlbuild.compiler.planner.main.commands._sql_test import build_test_command_plan from sqlbuild.compiler.planner.main.commands.audit import build_audit_command_plan from sqlbuild.compiler.planner.main.commands.seed import build_seed_command_plan from sqlbuild.compiler.planner.main.commands.seed_state import read_selected_seed_fingerprints -from sqlbuild.compiler.planner.main.commands.sql_test import build_test_command_plan from sqlbuild.compiler.planner.models import ( PlannerRelationsContext, PlannerScope, From 9180ed86c84e92412581c44c5c70f6c667658366 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:18:50 +0100 Subject: [PATCH 26/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 0eb4259082..633a9bc492 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 82955 +remaining_lines = 82303 [[stays]] path = "executor" From 43a85c692743a3e71c48289f7f687bc3c2b5897c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:40:40 +0100 Subject: [PATCH 27/37] refactor(rules): key, look up and store the custom-rule result cache natively Custom-rule identities (code identity, host runtime, rule metadata, source fingerprint, rules settings, dialect, project root), subject identities, read-set lookup, result merging, finding validation and ordering, and the cache file now live in sqlbuild-rules as pure functions behind a CustomRuleCache binding, with a new key format (custom-rules-native-v1), so the first run after upgrading misses once. Fact digests stay with the Python fact views the rule hosts read. A new integration test pins cold misses, warm hits, a hit through an unread model edit, and invalidation on a rule source edit. --- .../_helpers/rules/custom_rule_cache.rs | 267 +++++++++++ .../src/bindings/_helpers/rules/mod.rs | 1 + .../bindings/_helpers/rules/registration.rs | 4 +- .../src/custom_rule_cache/_helpers/current.rs | 25 + .../src/custom_rule_cache/_helpers/digest.rs | 27 ++ .../src/custom_rule_cache/_helpers/mod.rs | 3 + .../src/custom_rule_cache/_helpers/stored.rs | 53 +++ .../src/custom_rule_cache/constants.rs | 18 + .../src/custom_rule_cache/errors.rs | 5 + .../main/custom_rule_cache_text.rs | 60 +++ .../main/lookup_custom_rule_cache.rs | 54 +++ .../src/custom_rule_cache/main/mod.rs | 9 + .../main/ordered_custom_findings.rs | 20 + .../main/parse_custom_rule_cache.rs | 25 + .../main/pending_readsets.rs | 24 + .../custom_rule_cache/main/rule_identity.rs | 29 ++ .../custom_rule_cache/main/rule_subjects.rs | 35 ++ .../main/store_custom_results.rs | 67 +++ .../main/validate_custom_findings.rs | 61 +++ .../src/custom_rule_cache/mod.rs | 11 + .../src/custom_rule_cache/models.rs | 88 ++++ .../src/custom_rule_cache/tests.rs | 6 + .../src/custom_rule_cache/tests/helpers.rs | 53 +++ .../tests/test_custom_rule_cache.rs | 126 +++++ .../src/custom_rule_cache/tests/test_types.rs | 13 + .../src/custom_rule_cache/types.rs | 14 + crates/sqlbuild-rules/src/lib.rs | 1 + src/sqlbuild/_native.pyi | 49 +- .../_helpers/engine/custom_rules.py | 439 ++++-------------- src/sqlbuild/rule_engine/constants.py | 1 - src/sqlbuild/rule_engine/types.py | 3 + .../sqlbuild/cli/commands/main/_test_types.py | 9 + .../sqlbuild/cli/commands/main/test_rules.py | 60 +++ 33 files changed, 1303 insertions(+), 357 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/_helpers/current.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/_helpers/mod.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/_helpers/stored.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/constants.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/errors.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/main/custom_rule_cache_text.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/main/mod.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/main/ordered_custom_findings.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/main/parse_custom_rule_cache.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/main/pending_readsets.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/main/rule_identity.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/main/rule_subjects.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/main/store_custom_results.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/main/validate_custom_findings.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/mod.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/models.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/tests.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/tests/helpers.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/tests/test_types.rs create mode 100644 crates/sqlbuild-rules/src/custom_rule_cache/types.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs new file mode 100644 index 0000000000..07364fc1e3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -0,0 +1,267 @@ +//! Custom-rule result cache identities, read-set lookup, result merging and the cache file. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::{IntoPyObjectExt, Py, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use serde_json::Value; +use sqlbuild_rules::custom_rule_cache::main::custom_rule_cache_text::custom_rule_cache_text; +use sqlbuild_rules::custom_rule_cache::main::lookup_custom_rule_cache::lookup_custom_rule_cache; +use sqlbuild_rules::custom_rule_cache::main::ordered_custom_findings::ordered_custom_findings; +use sqlbuild_rules::custom_rule_cache::main::parse_custom_rule_cache::parse_custom_rule_cache; +use sqlbuild_rules::custom_rule_cache::main::pending_readsets::pending_readsets; +use sqlbuild_rules::custom_rule_cache::main::rule_identity::rule_identity; +use sqlbuild_rules::custom_rule_cache::main::rule_subjects::rule_subjects; +use sqlbuild_rules::custom_rule_cache::main::store_custom_results::store_custom_results; +use sqlbuild_rules::custom_rule_cache::main::validate_custom_findings::validate_custom_findings; +use sqlbuild_rules::custom_rule_cache::models::{ + CustomCacheLookup, CustomEvaluation, CustomRule, ModelSubjectInput, RuleIdentityInput, + SubjectsInput, +}; +use sqlbuild_rules::custom_rule_cache::types::{ReadSet, StoredRules, SubjectResults}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compiled_project::values::plain_value; + +const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERROR: &str = "RulesError"; + +/// `(code, subject, (read set, digest) or None, findings)` of one evaluated invocation. +type EvaluationPy<'py> = ( + String, + String, + Option<(ReadSet, Option)>, + Vec>, +); + +/// One custom rule's cache identity from its metadata, source fingerprint and settings. +#[pyfunction] +#[pyo3(signature = (**request))] +fn custom_rule_identity(request: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("custom_rule_identity", request)?; + Ok(rule_identity(&RuleIdentityInput { + code_identity: request.get("code_identity")?.extract()?, + runtime_version: request.get("runtime_version")?.extract()?, + code: request.get("code")?.extract()?, + family: request.get("family")?.extract()?, + slug: request.get("slug")?.extract()?, + message: request.get("message")?.extract()?, + remediation: request.get("remediation")?.extract()?, + subject: request.get("subject")?.extract()?, + implementation_fingerprint: request.get("implementation_fingerprint")?.extract()?, + config_json: plain_value(&request.get("config")?)?.to_string(), + dialect: request.get("dialect")?.extract()?, + project_dir: request.get("project_dir")?.extract()?, + })) +} + +fn python_value(py: Python<'_>, value: &Value) -> PyResult> { + match value { + Value::Null => Ok(py.None()), + Value::Bool(flag) => flag.into_py_any(py), + Value::Number(number) => match (number.as_i64(), number.as_u64()) { + (Some(integer), _) => integer.into_py_any(py), + (None, Some(integer)) => integer.into_py_any(py), + (None, None) => number.as_f64().unwrap_or(f64::NAN).into_py_any(py), + }, + Value::String(text) => text.into_py_any(py), + Value::Array(items) => { + let values: Vec> = items + .iter() + .map(|item| python_value(py, item)) + .collect::>()?; + PyList::new(py, values)?.into_py_any(py) + } + Value::Object(entries) => { + let dictionary: Bound<'_, PyDict> = PyDict::new(py); + for (key, item) in entries { + dictionary.set_item(key, python_value(py, item)?)?; + } + dictionary.into_py_any(py) + } + } +} + +fn readset_tuple<'py>(py: Python<'py>, readset: &ReadSet) -> PyResult> { + let keys: Vec> = readset + .iter() + .map(|key| PyTuple::new(py, key)) + .collect::>()?; + PyTuple::new(py, keys) +} + +fn rules_error(py: Python<'_>, message: String) -> PyResult { + let class: Bound<'_, PyAny> = py.import(RULES_ERRORS_MODULE)?.getattr(RULES_ERROR)?; + Ok(pyo3::PyErr::from_value(class.call1((message,))?)) +} + +fn stored_rules(path: &std::path::Path) -> StoredRules { + let Ok(text) = std::fs::read_to_string(path) else { + return StoredRules::new(); + }; + parse_custom_rule_cache(&text) +} + +/// One custom-rule run's subjects, reused and new results, and the cache file it reads and writes. +#[pyclass(module = "sqlbuild._native")] +struct CustomRuleCache { + path: Option, + rules: Vec, + project: SubjectsInput, + stored: StoredRules, + pending: Vec, + results: SubjectResults, + hits: usize, + misses: usize, +} + +#[pymethods] +impl CustomRuleCache { + /// Load the cache at `path` (`None` when caching is off) for `(code, identity, subject)` rules. + #[new] + #[pyo3(signature = (*, path, rules, target_name, models))] + fn new( + path: Option, + rules: Vec<(String, Option, Option)>, + target_name: Option, + models: Vec<(String, String, Option)>, + ) -> Self { + let project = SubjectsInput { + target_name, + models: models + .into_iter() + .map(|(path, name, materialized)| ModelSubjectInput { + path, + name, + materialized, + }) + .collect(), + }; + let mut rules: Vec = rules + .into_iter() + .map(|(code, identity, subject)| CustomRule { + subjects: rule_subjects(subject.as_deref(), &project), + code, + identity, + subject, + }) + .collect(); + rules.sort_by(|left, right| left.code.cmp(&right.code)); + let stored: StoredRules = path.as_deref().map(stored_rules).unwrap_or_default(); + let pending: Vec = pending_readsets(&stored, &rules); + Self { + path, + rules, + project, + stored, + pending, + results: SubjectResults::new(), + hits: 0, + misses: 0, + } + } + + /// Stored read sets whose current digest decides whether their results are reused. + fn pending_readsets<'py>(&self, py: Python<'py>) -> PyResult>> { + self.pending + .iter() + .map(|readset| readset_tuple(py, readset)) + .collect() + } + + /// Reuse results given each pending read set's digest; returns the subjects still to run. + #[pyo3(signature = (*, digests))] + fn plan(&mut self, digests: Vec>) -> BTreeMap>> { + if self.path.is_none() { + self.misses = self.rules.len(); + return self + .rules + .iter() + .map(|rule| (rule.code.clone(), None)) + .collect(); + } + let current: BTreeMap = self + .pending + .iter() + .zip(digests) + .filter_map(|(readset, digest)| Some((readset.clone(), digest?))) + .collect(); + let lookup: CustomCacheLookup = + lookup_custom_rule_cache(&self.stored, &self.rules, ¤t); + self.results.extend(lookup.hits); + self.hits = lookup.hit_count; + self.misses = lookup.miss_count; + lookup.plan + } + + /// Record evaluated results; with caching on, validate them, store them and write the file. + #[pyo3(signature = (*, evaluated))] + fn record(&mut self, py: Python<'_>, evaluated: Vec>) -> PyResult<()> { + let evaluations: Vec = evaluated + .into_iter() + .map(|(code, subject, cacheable, findings)| { + Ok(CustomEvaluation { + code, + subject, + cacheable, + findings: findings.iter().map(plain_value).collect::>()?, + }) + }) + .collect::>()?; + let Some(path) = self.path.clone() else { + self.extend_results(evaluations); + return Ok(()); + }; + validate_custom_findings(&self.rules, &self.project, &evaluations) + .map_err(|error| rules_error(py, error.0).unwrap_or_else(|error| error))?; + self.stored = store_custom_results(&self.stored, &self.rules, &evaluations); + self.extend_results(evaluations); + let text: String = custom_rule_cache_text(&self.stored, &self.rules); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let temporary: PathBuf = path.with_extension(format!( + "tmp-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + std::fs::write(&temporary, text)?; + std::fs::rename(&temporary, &path)?; + Ok(()) + } + + /// Every finding payload in rule code, then subject order. + fn findings(&self, py: Python<'_>) -> PyResult>> { + ordered_custom_findings(&self.rules, &self.results) + .iter() + .map(|finding| python_value(py, finding)) + .collect() + } + + #[getter] + fn hits(&self) -> usize { + self.hits + } + + #[getter] + fn misses(&self) -> usize { + self.misses + } +} + +impl CustomRuleCache { + fn extend_results(&mut self, evaluations: Vec) { + for evaluation in evaluations { + self.results + .insert((evaluation.code, evaluation.subject), evaluation.findings); + } + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(custom_rule_identity, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 9c49a5d267..7924011a1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod custom_rule_cache; pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; pub(crate) mod evaluation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index a1fbcc367e..23a5478684 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ - custom_rule_evidence, dsl_headers, evaluation, lint, results, rule_cache, skills, + custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, results, rule_cache, + skills, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -12,6 +13,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { dsl_headers::register(module)?; custom_rule_evidence::register(module)?; rule_cache::register(module)?; + custom_rule_cache::register(module)?; results::register(module)?; skills::register(module) } diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/current.rs b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/current.rs new file mode 100644 index 0000000000..19ab2197e4 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/current.rs @@ -0,0 +1,25 @@ +//! The stored results that still belong to the current rule and subject identities. + +use crate::custom_rule_cache::models::{CustomRule, StoredEntry, StoredRule}; +use crate::custom_rule_cache::types::StoredRules; + +/// The stored rule cache of `rule` when its identity is unchanged. +pub(crate) fn current_rule<'a>( + stored: &'a StoredRules, + rule: &CustomRule, +) -> Option<&'a StoredRule> { + let cached: &StoredRule = stored.get(&rule.code)?; + (rule.identity.as_deref() == Some(cached.identity.as_str())).then_some(cached) +} + +/// The stored entry for one subject when its identity is unchanged. +pub(crate) fn current_entry<'a>( + cached: &'a StoredRule, + key: &str, + identity: &str, +) -> Option<&'a StoredEntry> { + cached + .entries + .get(key) + .filter(|entry| entry.subject_identity == identity) +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs new file mode 100644 index 0000000000..720e15600a --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs @@ -0,0 +1,27 @@ +//! SHA-256 over length-prefixed fields, so no two field lists share an identity. + +use sha2::{Digest, Sha256}; + +use crate::constants::NATIVE_BUILD_IDENTITY; +use crate::custom_rule_cache::constants::CACHE_VERSION; + +/// The hex digest of the cache version, the native build and each field in order. +pub(crate) fn field_digest(kind: &str, fields: &[&str]) -> String { + let mut digest = Sha256::new(); + for field in [CACHE_VERSION, NATIVE_BUILD_IDENTITY, kind] + .iter() + .chain(fields.iter()) + { + digest.update((field.len() as u64).to_le_bytes()); + digest.update(field.as_bytes()); + } + format!("{:x}", digest.finalize()) +} + +/// An optional field as a presence flag and its text. +pub(crate) fn optional_fields(value: Option<&str>) -> [&str; 2] { + match value { + Some(text) => ["1", text], + None => ["0", ""], + } +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/mod.rs b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/mod.rs new file mode 100644 index 0000000000..4bd43b6a42 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod current; +pub(crate) mod digest; +pub(crate) mod stored; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/stored.rs b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/stored.rs new file mode 100644 index 0000000000..1f2457e97e --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/stored.rs @@ -0,0 +1,53 @@ +//! Stored rule caches decoded from their JSON form. + +use std::collections::BTreeMap; + +use serde_json::{Map, Value}; + +use crate::custom_rule_cache::constants::{ENTRIES_KEY, IDENTITY_KEY, READSETS_KEY}; +use crate::custom_rule_cache::models::{StoredEntry, StoredRule}; +use crate::custom_rule_cache::types::{FactKey, ReadSet}; + +fn strings(value: &Value) -> Option { + value + .as_array()? + .iter() + .map(|part| part.as_str().map(str::to_owned)) + .collect() +} + +fn readset(value: &Value) -> Option { + value.as_array()?.iter().map(strings).collect() +} + +fn entry(value: &Value) -> Option { + let [subject_identity, readset, reads_digest, findings] = value.as_array()?.as_slice() else { + return None; + }; + Some(StoredEntry { + subject_identity: subject_identity.as_str()?.to_owned(), + readset: readset.as_u64()? as usize, + reads_digest: reads_digest.as_str()?.to_owned(), + findings: findings.as_array()?.clone(), + }) +} + +pub(crate) fn stored_rule(value: &Value) -> Option { + let object: &Map = value.as_object()?; + let entries: BTreeMap = object + .get(ENTRIES_KEY)? + .as_object()? + .iter() + .map(|(subject, value)| Some((subject.clone(), entry(value)?))) + .collect::>()?; + Some(StoredRule { + identity: object.get(IDENTITY_KEY)?.as_str()?.to_owned(), + readsets: object + .get(READSETS_KEY)? + .as_array()? + .iter() + .map(readset) + .collect::>()?, + entries, + }) +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs b/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs new file mode 100644 index 0000000000..18d9b415af --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs @@ -0,0 +1,18 @@ +//! The custom-rule result cache format and the inputs its identities separate. + +/// Bumped whenever identity derivation or the stored entry shape changes. +pub(crate) const CACHE_VERSION: &str = "custom-rules-native-v1"; +pub(crate) const RULE_KIND: &str = "rule"; +pub(crate) const PROJECT_SUBJECT_KIND: &str = "project"; +pub(crate) const MODEL_SUBJECT_KIND: &str = "model"; +/// The subject key of a project-wide rule's single invocation. +pub(crate) const PROJECT_SUBJECT: &str = ""; +pub(crate) const PROJECT_SUBJECT_NAME: &str = "project"; +pub(crate) const MODEL_SUBJECT_NAME: &str = "model"; +pub(crate) const VERSION_KEY: &str = "version"; +pub(crate) const RULES_KEY: &str = "rules"; +pub(crate) const IDENTITY_KEY: &str = "identity"; +pub(crate) const READSETS_KEY: &str = "readsets"; +pub(crate) const ENTRIES_KEY: &str = "entries"; +pub(crate) const CODE_KEY: &str = "code"; +pub(crate) const PATH_KEY: &str = "path"; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/errors.rs b/crates/sqlbuild-rules/src/custom_rule_cache/errors.rs new file mode 100644 index 0000000000..0837df974a --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/errors.rs @@ -0,0 +1,5 @@ +//! Custom-rule results that contradict the rules that produced them. + +/// A host returned a finding outside the rule or subject it evaluated. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CustomFindingError(pub String); diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/custom_rule_cache_text.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/custom_rule_cache_text.rs new file mode 100644 index 0000000000..b41fac8c4d --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/custom_rule_cache_text.rs @@ -0,0 +1,60 @@ +//! `custom_rule_cache_text`: the stored form of the custom-rule result cache. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Map, Value, json}; + +use crate::custom_rule_cache::constants::{ + CACHE_VERSION, ENTRIES_KEY, IDENTITY_KEY, READSETS_KEY, RULES_KEY, VERSION_KEY, +}; +use crate::custom_rule_cache::models::{CustomRule, StoredRule}; +use crate::custom_rule_cache::types::StoredRules; + +fn rule_value(cached: &StoredRule, rule: &CustomRule) -> Value { + let live: BTreeSet<&str> = rule + .subjects + .iter() + .map(|subject| subject.key.as_str()) + .collect(); + let used: BTreeSet = cached + .entries + .iter() + .filter(|(key, _)| live.contains(key.as_str())) + .map(|(_, entry)| entry.readset) + .collect(); + let remap: BTreeMap = used + .iter() + .enumerate() + .map(|(new, old)| (*old, new)) + .collect(); + let entries: Map = cached + .entries + .iter() + .filter_map(|(key, entry)| { + let index: usize = *remap.get(&entry.readset)?; + let value = json!([ + entry.subject_identity, + index, + entry.reads_digest, + entry.findings + ]); + live.contains(key.as_str()).then(|| (key.clone(), value)) + }) + .collect(); + let readsets: Vec = used + .iter() + .filter_map(|index| cached.readsets.get(*index)) + .map(|readset| json!(readset)) + .collect(); + json!({ IDENTITY_KEY: cached.identity, READSETS_KEY: readsets, ENTRIES_KEY: entries }) +} + +/// Compact JSON of the format version and each selected rule's live subjects and read sets. +#[must_use] +pub fn custom_rule_cache_text(stored: &StoredRules, rules: &[CustomRule]) -> String { + let stored_rules: Map = rules + .iter() + .filter_map(|rule| Some((rule.code.clone(), rule_value(stored.get(&rule.code)?, rule)))) + .collect(); + json!({ VERSION_KEY: CACHE_VERSION, RULES_KEY: stored_rules }).to_string() +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs new file mode 100644 index 0000000000..7e05b58abf --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs @@ -0,0 +1,54 @@ +//! `lookup_custom_rule_cache`: reuse results whose recorded fact reads still digest the same. + +use std::collections::BTreeMap; + +use crate::custom_rule_cache::_helpers::current::{current_entry, current_rule}; +use crate::custom_rule_cache::models::{CustomCacheLookup, CustomRule, StoredEntry, StoredRule}; +use crate::custom_rule_cache::types::{ReadSet, StoredRules}; + +fn reusable<'a>( + cached: Option<&'a StoredRule>, + rule_subject: (&str, &str), + digests: &BTreeMap, +) -> Option<&'a StoredEntry> { + let cached: &StoredRule = cached?; + let entry: &StoredEntry = current_entry(cached, rule_subject.0, rule_subject.1)?; + let readset: &ReadSet = cached.readsets.get(entry.readset)?; + (digests.get(readset) == Some(&entry.reads_digest)).then_some(entry) +} + +/// Hits per rule and subject, and each rule's missing subjects (`None` when all miss). +#[must_use] +pub fn lookup_custom_rule_cache( + stored: &StoredRules, + rules: &[CustomRule], + digests: &BTreeMap, +) -> CustomCacheLookup { + let mut lookup = CustomCacheLookup::default(); + for rule in rules { + let cached: Option<&StoredRule> = current_rule(stored, rule); + let mut missing: Vec = Vec::new(); + for subject in &rule.subjects { + match reusable(cached, (&subject.key, &subject.identity), digests) { + Some(entry) => { + let key = (rule.code.clone(), subject.key.clone()); + lookup.hits.push((key, entry.findings.clone())); + lookup.hit_count += 1; + } + None => missing.push(subject.key.clone()), + } + } + if missing.is_empty() { + continue; + } + lookup.miss_count += missing.len(); + let planned: Option> = if missing.len() == rule.subjects.len() { + None + } else { + missing.sort(); + Some(missing) + }; + lookup.plan.insert(rule.code.clone(), planned); + } + lookup +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/mod.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/mod.rs new file mode 100644 index 0000000000..21874c954a --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/mod.rs @@ -0,0 +1,9 @@ +pub mod custom_rule_cache_text; +pub mod lookup_custom_rule_cache; +pub mod ordered_custom_findings; +pub mod parse_custom_rule_cache; +pub mod pending_readsets; +pub mod rule_identity; +pub mod rule_subjects; +pub mod store_custom_results; +pub mod validate_custom_findings; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/ordered_custom_findings.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/ordered_custom_findings.rs new file mode 100644 index 0000000000..4aee088c65 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/ordered_custom_findings.rs @@ -0,0 +1,20 @@ +//! `ordered_custom_findings`: custom findings in rule code, then subject order. + +use serde_json::Value; + +use crate::custom_rule_cache::models::CustomRule; +use crate::custom_rule_cache::types::SubjectResults; + +/// Every result's findings, rule by rule and subject by subject. +#[must_use] +pub fn ordered_custom_findings(rules: &[CustomRule], results: &SubjectResults) -> Vec { + let mut findings: Vec = Vec::new(); + for rule in rules { + for subject in &rule.subjects { + if let Some(values) = results.get(&(rule.code.clone(), subject.key.clone())) { + findings.extend(values.iter().cloned()); + } + } + } + findings +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/parse_custom_rule_cache.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/parse_custom_rule_cache.rs new file mode 100644 index 0000000000..61bc93d702 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/parse_custom_rule_cache.rs @@ -0,0 +1,25 @@ +//! `parse_custom_rule_cache`: the usable rule caches of a stored custom-rule cache. + +use serde_json::Value; + +use crate::custom_rule_cache::_helpers::stored::stored_rule; +use crate::custom_rule_cache::constants::{CACHE_VERSION, RULES_KEY, VERSION_KEY}; +use crate::custom_rule_cache::types::StoredRules; + +/// Rule caches written by this format; malformed rules are skipped and anything else is empty. +#[must_use] +pub fn parse_custom_rule_cache(text: &str) -> StoredRules { + let Ok(Value::Object(payload)) = serde_json::from_str::(text) else { + return StoredRules::new(); + }; + if payload.get(VERSION_KEY).and_then(Value::as_str) != Some(CACHE_VERSION) { + return StoredRules::new(); + } + let Some(Value::Object(rules)) = payload.get(RULES_KEY) else { + return StoredRules::new(); + }; + rules + .iter() + .filter_map(|(code, value)| Some((code.clone(), stored_rule(value)?))) + .collect() +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/pending_readsets.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/pending_readsets.rs new file mode 100644 index 0000000000..2b5af914b6 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/pending_readsets.rs @@ -0,0 +1,24 @@ +//! `pending_readsets`: the stored read sets whose current digest decides a cache hit. + +use crate::custom_rule_cache::_helpers::current::{current_entry, current_rule}; +use crate::custom_rule_cache::models::CustomRule; +use crate::custom_rule_cache::types::{ReadSet, StoredRules}; + +/// Distinct read sets of every reusable entry, in first-use order. +#[must_use] +pub fn pending_readsets(stored: &StoredRules, rules: &[CustomRule]) -> Vec { + let mut pending: Vec = Vec::new(); + for rule in rules { + let Some(cached) = current_rule(stored, rule) else { + continue; + }; + for subject in &rule.subjects { + let readset: Option<&ReadSet> = current_entry(cached, &subject.key, &subject.identity) + .and_then(|entry| cached.readsets.get(entry.readset)); + if let Some(readset) = readset.filter(|readset| !pending.contains(readset)) { + pending.push(readset.clone()); + } + } + } + pending +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_identity.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_identity.rs new file mode 100644 index 0000000000..987b8f9a80 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_identity.rs @@ -0,0 +1,29 @@ +//! `rule_identity`: everything besides fact reads that keys one custom rule's stored results. + +use crate::custom_rule_cache::_helpers::digest::{field_digest, optional_fields}; +use crate::custom_rule_cache::constants::RULE_KIND; +use crate::custom_rule_cache::models::RuleIdentityInput; + +/// A digest of the code identity, host runtime, rule metadata and source, config, dialect and root. +#[must_use] +pub fn rule_identity(input: &RuleIdentityInput) -> String { + let [has_subject, subject] = optional_fields(input.subject.as_deref()); + field_digest( + RULE_KIND, + &[ + &input.code_identity, + &input.runtime_version, + &input.code, + &input.family, + &input.slug, + &input.message, + &input.remediation, + has_subject, + subject, + &input.implementation_fingerprint, + &input.config_json, + &input.dialect, + &input.project_dir, + ], + ) +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_subjects.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_subjects.rs new file mode 100644 index 0000000000..46616300a8 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_subjects.rs @@ -0,0 +1,35 @@ +//! `rule_subjects`: the invocations one custom rule runs, with each subject's identity. + +use crate::custom_rule_cache::_helpers::digest::{field_digest, optional_fields}; +use crate::custom_rule_cache::constants::{ + MODEL_SUBJECT_KIND, PROJECT_SUBJECT, PROJECT_SUBJECT_KIND, PROJECT_SUBJECT_NAME, +}; +use crate::custom_rule_cache::models::{ModelSubjectInput, Subject, SubjectsInput}; + +/// The project subject for project-wide rules, otherwise every model sorted by path. +#[must_use] +pub fn rule_subjects(subject: Option<&str>, input: &SubjectsInput) -> Vec { + if subject == Some(PROJECT_SUBJECT_NAME) { + let [has_target, target] = optional_fields(input.target_name.as_deref()); + let count: String = input.models.len().to_string(); + return vec![Subject { + key: PROJECT_SUBJECT.to_owned(), + identity: field_digest(PROJECT_SUBJECT_KIND, &[&count, has_target, target]), + }]; + } + let mut models: Vec<&ModelSubjectInput> = input.models.iter().collect(); + models.sort_by(|left, right| left.path.cmp(&right.path)); + models + .into_iter() + .map(|model| { + let [has_materialized, materialized] = optional_fields(model.materialized.as_deref()); + Subject { + key: model.path.clone(), + identity: field_digest( + MODEL_SUBJECT_KIND, + &[&model.name, &model.path, has_materialized, materialized], + ), + } + }) + .collect() +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/store_custom_results.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/store_custom_results.rs new file mode 100644 index 0000000000..d313b2c242 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/store_custom_results.rs @@ -0,0 +1,67 @@ +//! `store_custom_results`: merge newly evaluated results into the stored rule caches. + +use std::collections::BTreeMap; + +use crate::custom_rule_cache::models::{CustomEvaluation, CustomRule, StoredEntry, StoredRule}; +use crate::custom_rule_cache::types::StoredRules; + +fn store_one( + mut current: StoredRule, + subject_identity: &str, + evaluation: &CustomEvaluation, +) -> StoredRule { + let Some((readset, Some(digest))) = &evaluation.cacheable else { + current.entries.remove(&evaluation.subject); + return current; + }; + let index: usize = match current.readsets.iter().position(|known| known == readset) { + Some(index) => index, + None => { + current.readsets.push(readset.clone()); + current.readsets.len() - 1 + } + }; + let entry = StoredEntry { + subject_identity: subject_identity.to_owned(), + readset: index, + reads_digest: digest.clone(), + findings: evaluation.findings.clone(), + }; + current.entries.insert(evaluation.subject.clone(), entry); + current +} + +/// Stored caches of the selected rules, with stale rule identities reset and new results stored. +#[must_use] +pub fn store_custom_results( + stored: &StoredRules, + rules: &[CustomRule], + evaluations: &[CustomEvaluation], +) -> StoredRules { + let mut updated = StoredRules::new(); + for rule in rules { + let Some(identity) = &rule.identity else { + continue; + }; + let mut current: StoredRule = stored + .get(&rule.code) + .filter(|previous| previous.identity == *identity) + .cloned() + .unwrap_or_else(|| StoredRule { + identity: identity.clone(), + ..StoredRule::default() + }); + let subjects: BTreeMap<&str, &str> = rule + .subjects + .iter() + .map(|subject| (subject.key.as_str(), subject.identity.as_str())) + .collect(); + for evaluation in evaluations.iter().filter(|item| item.code == rule.code) { + if let Some(subject_identity) = subjects.get(evaluation.subject.as_str()) { + current = store_one(current, subject_identity, evaluation); + } + } + updated.insert(rule.code.clone(), current); + } + updated +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/validate_custom_findings.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/validate_custom_findings.rs new file mode 100644 index 0000000000..de2fbf0025 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/validate_custom_findings.rs @@ -0,0 +1,61 @@ +//! `validate_custom_findings`: reject findings for other rules or unrelated model paths. + +use std::collections::BTreeSet; + +use serde_json::Value; + +use crate::custom_rule_cache::constants::{CODE_KEY, MODEL_SUBJECT_NAME, PATH_KEY}; +use crate::custom_rule_cache::errors::CustomFindingError; +use crate::custom_rule_cache::models::{CustomEvaluation, CustomRule, SubjectsInput}; + +fn shown(value: Option<&Value>) -> String { + match value { + Some(Value::String(text)) => text.clone(), + Some(other) => other.to_string(), + None => "None".to_owned(), + } +} + +/// An error for the first finding of an unselected rule or, for model rules, an unknown path. +pub fn validate_custom_findings( + rules: &[CustomRule], + project: &SubjectsInput, + evaluations: &[CustomEvaluation], +) -> Result<(), CustomFindingError> { + let codes: BTreeSet<&str> = rules.iter().map(|rule| rule.code.as_str()).collect(); + let model_rules: BTreeSet<&str> = rules + .iter() + .filter(|rule| rule.subject.as_deref() == Some(MODEL_SUBJECT_NAME)) + .map(|rule| rule.code.as_str()) + .collect(); + let paths: BTreeSet<&str> = project + .models + .iter() + .map(|model| model.path.as_str()) + .collect(); + for evaluation in evaluations { + for finding in &evaluation.findings { + let code: Option<&Value> = finding.get(CODE_KEY); + if !code + .and_then(Value::as_str) + .is_some_and(|code| codes.contains(code)) + { + return Err(CustomFindingError(format!( + "custom rule host returned a finding for unselected rule {}", + shown(code) + ))); + } + let path: Option<&Value> = finding.get(PATH_KEY); + let known: bool = path + .and_then(Value::as_str) + .is_some_and(|path| paths.contains(path)); + if model_rules.contains(evaluation.code.as_str()) && !known { + return Err(CustomFindingError(format!( + "model-subject custom rule returned a finding for unrelated path {}", + shown(path) + ))); + } + } + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/mod.rs b/crates/sqlbuild-rules/src/custom_rule_cache/mod.rs new file mode 100644 index 0000000000..6a67b87751 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/mod.rs @@ -0,0 +1,11 @@ +//! The custom-rule result cache: rule and subject identities, read-set lookup and the cache file. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/models.rs b/crates/sqlbuild-rules/src/custom_rule_cache/models.rs new file mode 100644 index 0000000000..d464c88238 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/models.rs @@ -0,0 +1,88 @@ +//! Custom-rule cache identity inputs, subjects, stored entries and evaluations. + +use std::collections::BTreeMap; + +use crate::custom_rule_cache::types::ReadSet; + +/// Everything besides recorded fact reads that changes one custom rule's results. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RuleIdentityInput { + pub code_identity: String, + pub runtime_version: String, + pub code: String, + pub family: String, + pub slug: String, + pub message: String, + pub remediation: String, + pub subject: Option, + pub implementation_fingerprint: String, + pub config_json: String, + pub dialect: String, + pub project_dir: String, +} + +/// One model as a custom-rule subject: path, name and materialization. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelSubjectInput { + pub path: String, + pub name: String, + pub materialized: Option, +} + +/// The project facts that decide which subjects custom rules run on. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SubjectsInput { + pub target_name: Option, + pub models: Vec, +} + +/// One rule invocation's subject: its key and an identity of the subject's own metadata. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Subject { + pub key: String, + pub identity: String, +} + +/// One selected custom rule, its identity when caching, subject kind and subjects. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CustomRule { + pub code: String, + pub identity: Option, + pub subject: Option, + pub subjects: Vec, +} + +/// One stored invocation result and the read set digest it was valid for. +#[derive(Clone, Debug, PartialEq)] +pub struct StoredEntry { + pub subject_identity: String, + pub readset: usize, + pub reads_digest: String, + pub findings: Vec, +} + +/// One rule's stored results, sharing read sets between subjects. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct StoredRule { + pub identity: String, + pub readsets: Vec, + pub entries: BTreeMap, +} + +/// Reused results, the subjects each rule must still evaluate (`None` for all) and counts. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CustomCacheLookup { + pub hits: Vec<((String, String), Vec)>, + pub plan: BTreeMap>>, + pub hit_count: usize, + pub miss_count: usize, +} + +/// One evaluated invocation, with its read set and digest when its result may be cached. +#[derive(Clone, Debug, PartialEq)] +pub struct CustomEvaluation { + pub code: String, + pub subject: String, + pub cacheable: Option<(ReadSet, Option)>, + pub findings: Vec, +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/tests.rs b/crates/sqlbuild-rules/src/custom_rule_cache/tests.rs new file mode 100644 index 0000000000..34532bb433 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_custom_rule_cache.rs"] +mod custom_rule_cache; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/tests/helpers.rs b/crates/sqlbuild-rules/src/custom_rule_cache/tests/helpers.rs new file mode 100644 index 0000000000..a90e902d58 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/tests/helpers.rs @@ -0,0 +1,53 @@ +use crate::custom_rule_cache::main::rule_subjects::rule_subjects; +use crate::custom_rule_cache::models::{ + CustomRule, ModelSubjectInput, RuleIdentityInput, SubjectsInput, +}; +use crate::custom_rule_cache::types::ReadSet; + +pub(crate) fn identity_input() -> RuleIdentityInput { + RuleIdentityInput { + code_identity: "code".to_owned(), + runtime_version: "runtime".to_owned(), + code: "CUSTOM001".to_owned(), + family: "CUSTOM".to_owned(), + slug: "orders-rule".to_owned(), + message: "message".to_owned(), + remediation: "remediation".to_owned(), + subject: Some("model".to_owned()), + implementation_fingerprint: "source-a".to_owned(), + config_json: "{}".to_owned(), + dialect: "duckdb".to_owned(), + project_dir: "/project".to_owned(), + } +} + +pub(crate) fn project(model_name: &str) -> SubjectsInput { + SubjectsInput { + target_name: Some("dev".to_owned()), + models: vec![ + ModelSubjectInput { + path: "models/orders.sql".to_owned(), + name: model_name.to_owned(), + materialized: None, + }, + ModelSubjectInput { + path: "models/customers.sql".to_owned(), + name: "customers".to_owned(), + materialized: Some("table".to_owned()), + }, + ], + } +} + +pub(crate) fn rule(model_name: &str) -> CustomRule { + CustomRule { + code: "CUSTOM001".to_owned(), + identity: Some("rule-identity".to_owned()), + subject: Some("model".to_owned()), + subjects: rule_subjects(Some("model"), &project(model_name)), + } +} + +pub(crate) fn readset() -> ReadSet { + vec![vec!["project.models".to_owned()]] +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs new file mode 100644 index 0000000000..807431d92f --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs @@ -0,0 +1,126 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::custom_rule_cache::main::custom_rule_cache_text::custom_rule_cache_text; +use crate::custom_rule_cache::main::lookup_custom_rule_cache::lookup_custom_rule_cache; +use crate::custom_rule_cache::main::parse_custom_rule_cache::parse_custom_rule_cache; +use crate::custom_rule_cache::main::pending_readsets::pending_readsets; +use crate::custom_rule_cache::main::rule_identity::rule_identity; +use crate::custom_rule_cache::main::store_custom_results::store_custom_results; +use crate::custom_rule_cache::models::{CustomEvaluation, CustomRule, RuleIdentityInput}; +use crate::custom_rule_cache::tests::helpers::{identity_input, readset, rule}; +use crate::custom_rule_cache::tests::test_types::{IdentityTestCase, LookupTestCase}; +use crate::custom_rule_cache::types::ReadSet; + +#[test] +fn given_rule_identity_inputs_when_one_changes_then_only_relevant_edits_change_the_identity() { + let test_cases = [ + IdentityTestCase { + description: "the same inputs give the same identity", + edit: |_| {}, + expected_changed: false, + }, + IdentityTestCase { + description: "an edited rule source changes the identity", + edit: |input| input.implementation_fingerprint = "source-b".to_owned(), + expected_changed: true, + }, + IdentityTestCase { + description: "edited rules configuration changes the identity", + edit: |input| input.config_json = "{\"select\":[]}".to_owned(), + expected_changed: true, + }, + IdentityTestCase { + description: "a project-wide subject changes the identity", + edit: |input| input.subject = Some("project".to_owned()), + expected_changed: true, + }, + ]; + + for test_case in test_cases { + let mut edited: RuleIdentityInput = identity_input(); + (test_case.edit)(&mut edited); + let changed: bool = rule_identity(&edited) != rule_identity(&identity_input()); + + assert_eq!( + changed, test_case.expected_changed, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_results_when_looking_up_then_unchanged_subjects_and_reads_hit() { + let test_cases = [ + LookupTestCase { + description: "unchanged reads and subjects hit every subject", + current_digest: "digest-a", + model_name: "orders", + expected_plan: vec![], + expected_hits: 2, + }, + LookupTestCase { + description: "a changed fact digest misses every subject", + current_digest: "digest-b", + model_name: "orders", + expected_plan: vec![("CUSTOM001", None)], + expected_hits: 0, + }, + LookupTestCase { + description: "a renamed model misses only that subject", + current_digest: "digest-a", + model_name: "orders_v2", + expected_plan: vec![("CUSTOM001", Some(vec!["models/orders.sql"]))], + expected_hits: 1, + }, + ]; + + for test_case in test_cases { + let stored_rule: CustomRule = rule("orders"); + let evaluations: Vec = stored_rule + .subjects + .iter() + .map(|subject| CustomEvaluation { + code: stored_rule.code.clone(), + subject: subject.key.clone(), + cacheable: Some((readset(), Some("digest-a".to_owned()))), + findings: vec![json!({"code": "CUSTOM001", "path": subject.key})], + }) + .collect(); + let text: String = custom_rule_cache_text( + &store_custom_results( + &BTreeMap::new(), + std::slice::from_ref(&stored_rule), + &evaluations, + ), + &[stored_rule], + ); + let current: CustomRule = rule(test_case.model_name); + let stored = parse_custom_rule_cache(&text); + let digests: BTreeMap = + pending_readsets(&stored, std::slice::from_ref(¤t)) + .into_iter() + .map(|readset| (readset, test_case.current_digest.to_owned())) + .collect(); + let lookup = lookup_custom_rule_cache(&stored, &[current], &digests); + let expected_plan: BTreeMap>> = test_case + .expected_plan + .iter() + .map(|(code, subjects)| { + let subjects = subjects + .as_ref() + .map(|items| items.iter().map(|item| (*item).to_owned()).collect()); + ((*code).to_owned(), subjects) + }) + .collect(); + + assert_eq!(lookup.plan, expected_plan, "{}", test_case.description); + assert_eq!( + lookup.hit_count, test_case.expected_hits, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_types.rs b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_types.rs new file mode 100644 index 0000000000..d09156ee8c --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_types.rs @@ -0,0 +1,13 @@ +pub(crate) struct IdentityTestCase { + pub(crate) description: &'static str, + pub(crate) edit: fn(&mut crate::custom_rule_cache::models::RuleIdentityInput), + pub(crate) expected_changed: bool, +} + +pub(crate) struct LookupTestCase { + pub(crate) description: &'static str, + pub(crate) current_digest: &'static str, + pub(crate) model_name: &'static str, + pub(crate) expected_plan: Vec<(&'static str, Option>)>, + pub(crate) expected_hits: usize, +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/types.rs b/crates/sqlbuild-rules/src/custom_rule_cache/types.rs new file mode 100644 index 0000000000..2c2458faa4 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/types.rs @@ -0,0 +1,14 @@ +//! Recorded fact reads and stored custom-rule caches. + +use std::collections::BTreeMap; + +use crate::custom_rule_cache::models::StoredRule; + +/// One compiler fact a custom rule read, as its key parts. +pub type FactKey = Vec; +/// The sorted, distinct facts one invocation read. +pub type ReadSet = Vec; +/// Every usable stored rule cache by rule code. +pub type StoredRules = BTreeMap; +/// Results by `(rule code, subject key)`, each a list of finding payloads. +pub type SubjectResults = BTreeMap<(String, String), Vec>; diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index d6754baad4..11d313a627 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -4,6 +4,7 @@ pub mod configuration; pub mod constants; +pub mod custom_rule_cache; pub mod dsl_headers; pub mod engine; pub mod errors; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index dac200dbd3..d29993b5bf 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -253,7 +253,7 @@ from sqlbuild.presentation.classes.transient_status_reporter import TransientSta from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.rule_engine.models import Finding, Rule, RulesResult +from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import ( @@ -3218,6 +3218,53 @@ def custom_rule_import_closure(*, rule: Rule, project_dir: Path) -> tuple[Path, def custom_rule_implementation_fingerprint( *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] | None = None ) -> str: ... +def custom_rule_identity( + *, + code_identity: str, + runtime_version: str, + code: str, + family: str, + slug: str, + message: str, + remediation: str, + subject: str | None, + implementation_fingerprint: str, + config: RulesConfig, + dialect: str, + project_dir: str, +) -> str: ... + +class CustomRuleCache: + """One custom-rule run's subjects, reused and new results, and its cache file.""" + + def __init__( + self, + *, + path: Path | None, + rules: list[tuple[str, str | None, str | None]], + target_name: str | None, + models: list[tuple[str, str, str | None]], + ) -> None: ... + def pending_readsets(self) -> list[tuple[tuple[str, ...], ...]]: ... + def plan(self, *, digests: list[str | None]) -> dict[str, list[str] | None]: ... + def record( + self, + *, + evaluated: list[ + tuple[ + str, + str, + tuple[tuple[tuple[str, ...], ...], str | None] | None, + list[dict[str, object]], + ] + ], + ) -> None: ... + def findings(self) -> list[dict[str, object]]: ... + @property + def hits(self) -> int: ... + @property + def misses(self) -> int: ... + def custom_rule_test_evidence( *, rule: Rule, project_dir: Path ) -> list[tuple[str, int, int, str]]: ... diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index fd3d9d5d8e..140c20c2cd 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -2,19 +2,17 @@ from __future__ import annotations -import hashlib import os -import threading import time from collections.abc import Callable -from dataclasses import asdict, dataclass, replace +from dataclasses import replace from pathlib import Path from typing import Any -import orjson - from sqlbuild._native import ( + CustomRuleCache, compiled_project_record, + custom_rule_identity, custom_rule_implementation_fingerprint, custom_rule_import_closure, ) @@ -23,50 +21,17 @@ from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine.fact_replay import full_fact_keys, is_tree_fact -from sqlbuild.rule_engine._helpers.host.custom_host_pool import ( - decode_readset, - finding_payload, - run_custom_hosts, -) +from sqlbuild.rule_engine._helpers.host.custom_host_pool import run_custom_hosts from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.classes.fact_digests import FactDigests from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.rule_engine.constants import ( CUSTOM_HOST_RUNTIME_VERSION, - CUSTOM_RULE_PROJECT_SUBJECT, CUSTOM_RULES_CACHE_FILE, - CUSTOM_RULES_CACHE_VERSION, -) -from sqlbuild.rule_engine.exceptions import FactDigestError, RulesError -from sqlbuild.rule_engine.models import ( - CustomHostEvaluation, - CustomHostRun, - CustomRulesOutcome, - Rule, - RulesConfig, ) -from sqlbuild.rule_engine.types import FactKey, RuleSubject - - -@dataclass(frozen=True) -class _Subject: - key: str - identity: str - - -@dataclass(frozen=True) -class _CachedEntry: - subject_identity: str - readset: int - reads_digest: str - findings: tuple[dict[str, object], ...] - - -@dataclass -class _RuleCache: - identity: str - readsets: list[tuple[FactKey, ...]] - entries: dict[str, _CachedEntry] +from sqlbuild.rule_engine.exceptions import FactDigestError +from sqlbuild.rule_engine.models import CustomHostRun, CustomRulesOutcome, Rule, RulesConfig +from sqlbuild.rule_engine.types import CustomEvaluationRow, FactKey def evaluate_custom_rules_cached( @@ -87,75 +52,43 @@ def evaluate_custom_rules_cached( if not custom: return CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) facts_project: CompiledProjectView = host_project(project) - if not config.cache.enabled: - uncached: CustomHostRun = run_custom_hosts( - project=facts_project, - config=config, - project_dir=project_dir, - dialect=dialect, - rules=custom, - model_paths=_model_paths(project), - plan={rule.code: None for rule in custom}, - track_reads=False, - verify_determinism=verify_determinism, - ) - return CustomRulesOutcome( - findings=_ordered_findings( - rules=custom, - subjects=_subjects(project=project, rules=custom), - results={key: value.findings for key, value in uncached.evaluated.items()}, - ), - cache_hits=0, - cache_misses=len(custom), - custom_ms=_elapsed_ms(started), - custom_cpu_ms=_cpu_ms(started_cpu), - ) + enabled: bool = config.cache.enabled + cache: CustomRuleCache = CustomRuleCache( + path=( + rules_bulk_cache_path(project_dir=project_dir, file_name=CUSTOM_RULES_CACHE_FILE) + if enabled + else None + ), + rules=[ + ( + rule.code, + _rule_identity( + rule=rule, + config=config, + project_dir=project_dir, + dialect=dialect, + fingerprint=(implementation_fingerprints or {}).get(rule.code), + ) + if enabled + else None, + None if rule.subject is None else rule.subject.value, + ) + for rule in custom + ], + target_name=project.effective_target_name, + models=[ + (model.relative_path.as_posix(), model.name, _materialization(model)) + for model in project.models + ], + ) digests: FactDigests = FactDigests( views=_views_factory(project=facts_project, project_dir=project_dir, dialect=dialect) ) - subjects: dict[str, tuple[_Subject, ...]] = _subjects(project=project, rules=custom) - identities: dict[str, str] = { - rule.code: _rule_identity( - rule=rule, - config=config, - project_dir=project_dir, - dialect=dialect, - fingerprint=(implementation_fingerprints or {}).get(rule.code), - ) - for rule in custom - } - cache_path: Path = rules_bulk_cache_path( - project_dir=project_dir, file_name=CUSTOM_RULES_CACHE_FILE - ) - stored: dict[str, _RuleCache] = _read_cache(cache_path) - results: dict[tuple[str, str], tuple[dict[str, object], ...]] = {} - plan: dict[str, list[str] | None] = {} - hits: int = 0 - for rule in custom: - cached: _RuleCache | None = stored.get(rule.code) - if cached is not None and cached.identity != identities[rule.code]: - cached = None - current_digests: dict[int, str | None] = {} - missing: list[str] = [] - for subject in subjects[rule.code]: - entry: _CachedEntry | None = None if cached is None else cached.entries.get(subject.key) - if ( - entry is not None - and cached is not None - and entry.subject_identity == subject.identity - and _readset_digest( - cached=cached, index=entry.readset, digests=digests, memo=current_digests - ) - == entry.reads_digest - ): - results[(rule.code, subject.key)] = entry.findings - hits += 1 - else: - missing.append(subject.key) - if missing: - plan[rule.code] = None if len(missing) == len(subjects[rule.code]) else sorted(missing) - misses: int = sum( - len(subjects[code]) if planned is None else len(planned) for code, planned in plan.items() + plan: dict[str, list[str] | None] = cache.plan( + digests=[ + _readset_digest(readset=readset, digests=digests) + for readset in cache.pending_readsets() + ] ) if plan: run: CustomHostRun = run_custom_hosts( @@ -164,30 +97,16 @@ def evaluate_custom_rules_cached( project_dir=project_dir, dialect=dialect, rules=custom, - model_paths=_model_paths(project), + model_paths=tuple(sorted(model.relative_path.as_posix() for model in project.models)), plan=plan, - track_reads=True, + track_reads=enabled, verify_determinism=verify_determinism, ) - _validate_paths(project=project, rules=custom, evaluated=run.evaluated) - stored = _store( - stored=stored, - identities=identities, - subjects=subjects, - run=run, - digests=digests, - ) - for key, value in run.evaluated.items(): - results[key] = value.findings - _write_cache(path=cache_path, stored=stored, subjects=subjects) + cache.record(evaluated=_evaluations(run=run, digests=digests, cacheable=enabled)) return CustomRulesOutcome( - findings=_ordered_findings( - rules=custom, - subjects=subjects, - results=results, - ), - cache_hits=hits, - cache_misses=misses, + findings=tuple(cache.findings()), + cache_hits=cache.hits, + cache_misses=cache.misses, custom_ms=_elapsed_ms(started), custom_cpu_ms=_cpu_ms(started_cpu), ) @@ -202,38 +121,6 @@ def build() -> RuleFactViews: return build -def _subjects( - *, project: CompiledProjectView, rules: tuple[Rule, ...] -) -> dict[str, tuple[_Subject, ...]]: - project_subject: tuple[_Subject, ...] = ( - _Subject( - key=CUSTOM_RULE_PROJECT_SUBJECT, - identity=_digest([len(project.models), project.effective_target_name]), - ), - ) - model_subjects: tuple[_Subject, ...] = ( - tuple( - _Subject( - key=model.relative_path.as_posix(), - identity=_digest( - [model.name, model.relative_path.as_posix(), _materialization(model)] - ), - ) - for model in sorted(project.models, key=lambda item: item.relative_path.as_posix()) - ) - if any(rule.subject is not RuleSubject.PROJECT for rule in rules) - else () - ) - return { - rule.code: project_subject if rule.subject is RuleSubject.PROJECT else model_subjects - for rule in rules - } - - -def _model_paths(project: CompiledProjectView) -> tuple[str, ...]: - return tuple(sorted(model.relative_path.as_posix() for model in project.models)) - - def _materialization(model: Any) -> str | None: value: object = model.config.values.get("materialized") return value if isinstance(value, str) else None @@ -242,44 +129,33 @@ def _materialization(model: Any) -> str | None: def _rule_identity( *, rule: Rule, config: RulesConfig, project_dir: Path, dialect: str, fingerprint: str | None ) -> str: - return _digest( - [ - CUSTOM_RULES_CACHE_VERSION, - CUSTOM_HOST_RUNTIME_VERSION, - compiled_code_identity(), - rule.code, - rule.family, - rule.slug, - rule.message, - rule.remediation, - None if rule.subject is None else rule.subject.value, - fingerprint - if fingerprint is not None - else custom_rule_implementation_fingerprint( - rule=rule, - project_dir=project_dir, - import_closure=custom_rule_import_closure(rule=rule, project_dir=project_dir), - ), - asdict(config), - dialect, - str(project_dir.resolve()), - ] + return custom_rule_identity( + code_identity=compiled_code_identity(), + runtime_version=CUSTOM_HOST_RUNTIME_VERSION, + code=rule.code, + family=rule.family, + slug=rule.slug, + message=rule.message, + remediation=rule.remediation, + subject=None if rule.subject is None else rule.subject.value, + implementation_fingerprint=fingerprint + if fingerprint is not None + else custom_rule_implementation_fingerprint( + rule=rule, + project_dir=project_dir, + import_closure=custom_rule_import_closure(rule=rule, project_dir=project_dir), + ), + config=config, + dialect=dialect, + project_dir=str(project_dir.resolve()), ) -def _readset_digest( - *, cached: _RuleCache, index: int, digests: FactDigests, memo: dict[int, str | None] -) -> str | None: - if index not in memo: - try: - memo[index] = ( - digests.combined(cached.readsets[index]) - if 0 <= index < len(cached.readsets) - else None - ) - except FactDigestError: - memo[index] = None - return memo[index] +def _readset_digest(*, readset: tuple[FactKey, ...], digests: FactDigests) -> str | None: + try: + return digests.combined(readset) + except FactDigestError: + return None def host_project(project: CompiledProjectView) -> CompiledProject: @@ -302,91 +178,30 @@ def host_project(project: CompiledProjectView) -> CompiledProject: ) -def _validate_paths( - *, - project: CompiledProjectView, - rules: tuple[Rule, ...], - evaluated: dict[tuple[str, str], CustomHostEvaluation], -) -> None: - codes: frozenset[str] = frozenset(rule.code for rule in rules) - model_rules: frozenset[str] = frozenset( - rule.code for rule in rules if rule.subject is RuleSubject.MODEL - ) - model_paths: frozenset[str] = frozenset( - model.relative_path.as_posix() for model in project.models - ) - for (code, _subject), result in evaluated.items(): - for finding in result.findings: - if finding.get("code") not in codes: - raise RulesError( - f"custom rule host returned a finding for unselected rule {finding.get('code')}" - ) - if code in model_rules and finding.get("path") not in model_paths: - raise RulesError( - "model-subject custom rule returned a finding for unrelated path " - f"{finding.get('path')}" - ) - +def _evaluations( + *, run: CustomHostRun, digests: FactDigests, cacheable: bool +) -> list[CustomEvaluationRow]: + """Each evaluated invocation with its read set and its digest when it may be cached.""" -def _store( - *, - stored: dict[str, _RuleCache], - identities: dict[str, str], - subjects: dict[str, tuple[_Subject, ...]], - run: CustomHostRun, - digests: FactDigests, -) -> dict[str, _RuleCache]: - updated: dict[str, _RuleCache] = {} full_keys: tuple[FactKey, ...] | None = None - for code, identity in identities.items(): - previous: _RuleCache | None = stored.get(code) - current: _RuleCache = ( - _RuleCache(identity=identity, readsets=[], entries={}) - if previous is None or previous.identity != identity - else _RuleCache( - identity=identity, - readsets=list(previous.readsets), - entries=dict(previous.entries), - ) - ) - updated[code] = current - by_subject: dict[str, _Subject] = {subject.key: subject for subject in subjects[code]} - positions: dict[tuple[FactKey, ...], int] = { - readset: index for index, readset in enumerate(current.readsets) - } - combined_by_readset: dict[tuple[FactKey, ...], str | None] = {} - for (evaluated_code, subject_key), result in run.evaluated.items(): - if evaluated_code != code: - continue - if code in run.uncacheable: - current.entries.pop(subject_key, None) - continue + combined: dict[tuple[FactKey, ...], str | None] = {} + rows: list[CustomEvaluationRow] = [] + for (code, subject), result in run.evaluated.items(): + stored: tuple[tuple[FactKey, ...], str | None] | None = None + if cacheable and code not in run.uncacheable: reads: tuple[FactKey, ...] | None = result.reads if reads is None: if full_keys is None: full_keys = full_fact_keys(digests.views) reads = (*run.untracked_reads.get(code, ()), *full_keys) readset: tuple[FactKey, ...] = tuple(sorted(set(reads))) - if readset not in combined_by_readset: - combined_by_readset[readset] = _observed_digest( + if readset not in combined: + combined[readset] = _observed_digest( readset=readset, observed=run.observed, digests=digests ) - combined: str | None = combined_by_readset[readset] - if combined is None: - current.entries.pop(subject_key, None) - continue - index: int | None = positions.get(readset) - if index is None: - index = len(current.readsets) - current.readsets.append(readset) - positions[readset] = index - current.entries[subject_key] = _CachedEntry( - subject_identity=by_subject[subject_key].identity, - readset=index, - reads_digest=combined, - findings=result.findings, - ) - return updated + stored = (readset, combined[readset]) + rows.append((code, subject, stored, list(result.findings))) + return rows def _observed_digest( @@ -403,90 +218,6 @@ def _observed_digest( return None -def _ordered_findings( - *, - rules: tuple[Rule, ...], - subjects: dict[str, tuple[_Subject, ...]], - results: dict[tuple[str, str], tuple[dict[str, object], ...]], -) -> tuple[dict[str, object], ...]: - findings: list[dict[str, object]] = [] - for rule in rules: - for subject in subjects[rule.code]: - findings.extend(results.get((rule.code, subject.key), ())) - return tuple(findings) - - -def _read_cache(path: Path) -> dict[str, _RuleCache]: - try: - payload: object = orjson.loads(path.read_bytes()) - except (OSError, orjson.JSONDecodeError): - return {} - if not isinstance(payload, dict) or payload.get("version") != CUSTOM_RULES_CACHE_VERSION: - return {} - rules: object = payload.get("rules") - if not isinstance(rules, dict): - return {} - stored: dict[str, _RuleCache] = {} - for code, value in rules.items(): - try: - stored[str(code)] = _RuleCache( - identity=str(value["identity"]), - readsets=list(map(decode_readset, value["readsets"])), - entries={ - str(subject): _decode_entry(entry) - for subject, entry in value["entries"].items() - }, - ) - except (KeyError, TypeError, ValueError, IndexError): - continue - return stored - - -def _write_cache( - *, path: Path, stored: dict[str, _RuleCache], subjects: dict[str, tuple[_Subject, ...]] -) -> None: - rules: dict[str, object] = {} - for code, cached in stored.items(): - if code not in subjects: - continue - live: frozenset[str] = frozenset(subject.key for subject in subjects[code]) - entries: dict[str, _CachedEntry] = { - key: entry for key, entry in cached.entries.items() if key in live - } - used: list[int] = sorted({entry.readset for entry in entries.values()}) - remap: dict[int, int] = {old: new for new, old in enumerate(used)} - rules[code] = { - "identity": cached.identity, - "readsets": [list(map(list, cached.readsets[index])) for index in used], - "entries": { - key: [ - entry.subject_identity, - remap[entry.readset], - entry.reads_digest, - list(entry.findings), - ] - for key, entry in entries.items() - }, - } - path.parent.mkdir(parents=True, exist_ok=True) - temporary: Path = path.with_suffix(f".tmp-{os.getpid()}-{threading.get_ident()}") - temporary.write_bytes(orjson.dumps({"version": CUSTOM_RULES_CACHE_VERSION, "rules": rules})) - temporary.replace(path) - - -def _decode_entry(value: Any) -> _CachedEntry: - return _CachedEntry( - subject_identity=str(value[0]), - readset=int(value[1]), - reads_digest=str(value[2]), - findings=tuple(map(finding_payload, value[3])), - ) - - -def _digest(value: object) -> str: - return hashlib.sha256(orjson.dumps(value, default=str)).hexdigest() - - def _cpu_seconds() -> float: """This thread's CPU plus that of finished child processes, which run the rule hosts.""" diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index b9f34c04ad..49fe4a4a5e 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -11,7 +11,6 @@ CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v2" CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 CUSTOM_RULE_PROJECT_SUBJECT: str = "" -CUSTOM_RULES_CACHE_VERSION: str = "custom-rules-reads-v1" RULES_CACHE_DIRECTORY_NAME: str = "rules-cache" RULES_BULK_CACHE_DIRECTORY_NAME: str = "bulk" CUSTOM_RULES_CACHE_FILE: str = "custom-rules.json" diff --git a/src/sqlbuild/rule_engine/types.py b/src/sqlbuild/rule_engine/types.py index cc82fc5b28..baf5012398 100644 --- a/src/sqlbuild/rule_engine/types.py +++ b/src/sqlbuild/rule_engine/types.py @@ -27,6 +27,9 @@ type RuleOptionValue = bool | int | str | tuple[str, ...] | tuple[int, ...] type FactKey = tuple[str, ...] type FindingRow = tuple[bool, str, str, int, int, str, str] +type CustomEvaluationRow = tuple[ + str, str, tuple[tuple[FactKey, ...], str | None] | None, list[dict[str, object]] +] type CustomRulePlan = dict[str, frozenset[str] | None] type CustomHostPlan = dict[str, list[str] | None] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 791f88a676..d4cc7e7c3b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1089,3 +1089,12 @@ class SqlRuleCacheSequenceTestCase: description: str steps: tuple[tuple[str, str], ...] expected_hits_and_misses: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class CustomRuleCacheSequenceTestCase: + """Compiles after each `(rule source, model SQL)` step and the cache counts each reports.""" + + description: str + steps: tuple[tuple[str, str], ...] + expected_hits_and_misses: tuple[tuple[int, int], ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index fc513adc9c..40506fd311 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -29,6 +29,7 @@ EarlyLintInputsTestCase, ExplicitContractOutputRuleIntegrationTestCase, ImplicitAliasRuleIntegrationTestCase, + CustomRuleCacheSequenceTestCase, NumericRangeDecisionIntegrationTestCase, RulePassIntegrationTestCase, RulesIntegrationTestCase, @@ -50,7 +51,22 @@ ) _CACHE_MODEL_SQL: str = 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n' _CACHE_EDITED_SQL: str = _CACHE_MODEL_SQL.replace("LIMIT 1", "LIMIT 2") +_CUSTOM_CACHE_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["XSQBRCACHE001"]\n' +) +_CUSTOM_CACHE_RULE: str = """from sqlbuild.rules import Finding, Model, RuleContext, rule +@rule( + code="XSQBRCACHE001", + message="Models must live in the final directory", + remediation="Move this model beneath models/final/.", +) +def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: + return [] if "final" in model.path.parts else [ctx.finding(subject=model)] +""" +_CUSTOM_CACHE_EDITED_RULE: str = _CUSTOM_CACHE_RULE.replace('"final" in', '"staging" in') +_CUSTOM_CACHE_MODEL_SQL: str = 'MODEL (description "Orders");\nSELECT 1 AS order_id\n' +_CUSTOM_CACHE_EDITED_SQL: str = _CUSTOM_CACHE_MODEL_SQL.replace("SELECT 1", "SELECT 2") _EARLY_LINT_STOP_TIMEOUT_S: float = 30.0 _EARLY_LINT_FINDINGS: tuple[tuple[str, str, int | None, int | None], ...] = ( @@ -2847,3 +2863,47 @@ def test_given_sql_rule_cache_when_inputs_change_then_only_unchanged_runs_hit( observed.append((timings["rule_cache_hits"], timings["rule_cache_misses"])) assert tuple(observed) == test_case.expected_hits_and_misses + + +@pytest.mark.parametrize( + "test_case", + [ + CustomRuleCacheSequenceTestCase( + description=( + "the custom rule misses cold and after a source edit, and hits through a model " + "SQL edit it never read; the built-in model rules add one count per run" + ), + steps=( + (_CUSTOM_CACHE_RULE, _CUSTOM_CACHE_MODEL_SQL), + (_CUSTOM_CACHE_RULE, _CUSTOM_CACHE_MODEL_SQL), + (_CUSTOM_CACHE_RULE, _CUSTOM_CACHE_EDITED_SQL), + (_CUSTOM_CACHE_EDITED_RULE, _CUSTOM_CACHE_EDITED_SQL), + (_CUSTOM_CACHE_EDITED_RULE, _CUSTOM_CACHE_EDITED_SQL), + (_CUSTOM_CACHE_RULE, _CUSTOM_CACHE_EDITED_SQL), + ), + expected_hits_and_misses=((0, 2), (2, 0), (1, 1), (0, 2), (2, 0), (0, 2)), + ) + ], + ids=lambda case: case.description, +) +def test_given_custom_rule_cache_when_inputs_change_then_only_unchanged_runs_hit( + test_case: CustomRuleCacheSequenceTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(_CUSTOM_CACHE_TOML, encoding="utf-8") + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + rule_file: Path = tmp_path / "rules" / "layout.py" + rule_file.parent.mkdir() + observed: list[tuple[int, int]] = [] + for rule_source, model_sql in test_case.steps: + rule_file.write_text(rule_source, encoding="utf-8") + model.write_text(model_sql, encoding="utf-8") + _ = main(["--project-dir", str(tmp_path), "compile", "--json"]) + timings: dict[str, int] = cast( + dict[str, int], json.loads(capsys.readouterr().out)["compile_timings"] + ) + observed.append((timings["rule_cache_hits"], timings["rule_cache_misses"])) + + assert tuple(observed) == test_case.expected_hits_and_misses From 0a7d0b664d391ee2f098622867fedbb9e74c07a6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:49:24 +0100 Subject: [PATCH 28/37] refactor(rules): load rules configuration natively The native loader now returns the RulesConfig models directly and raises RulesError or RulesConfigError itself; the allow_exceptions policy check moves to sqlbuild-rules. The JSON configuration hand-off and its Python normalisation are removed. Selector resolution against the compiled project stays with the planner scope in Python. --- .../src/bindings/_helpers/rules/evaluation.rs | 9 - .../src/bindings/_helpers/rules/mod.rs | 1 + .../bindings/_helpers/rules/registration.rs | 3 +- .../bindings/_helpers/rules/rules_config.rs | 263 ++++++++++++++++++ .../src/configuration/_helpers/loading.rs | 5 - .../src/configuration/errors.rs | 8 + .../main/forbidden_exceptions.rs | 57 ++++ .../src/configuration/main/load.rs | 5 - .../configuration/main/load_rules_config.rs | 10 + .../src/configuration/main/mod.rs | 3 +- .../sqlbuild-rules/src/configuration/mod.rs | 1 + src/sqlbuild/_native.pyi | 2 +- .../rule_engine/_helpers/engine/config.py | 197 +------------ .../rule_engine/_helpers/engine/native.py | 12 - .../_helpers/engine/rule_harness.py | 4 +- src/sqlbuild/rule_engine/constants.py | 4 - src/sqlbuild/rule_engine/main/load_config.py | 4 +- 17 files changed, 352 insertions(+), 236 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs create mode 100644 crates/sqlbuild-rules/src/configuration/errors.rs create mode 100644 crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs delete mode 100644 crates/sqlbuild-rules/src/configuration/main/load.rs create mode 100644 crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs index cb5c126936..188c8977dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs @@ -2,7 +2,6 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_rules::configuration::main::load; use sqlbuild_rules::constants::API_VERSION; use sqlbuild_rules::models::CatalogueResponse; use sqlbuild_rules::rules::main::{catalogue, selected_codes}; @@ -18,13 +17,6 @@ fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { .map_err(value_error) } -#[pyfunction] -fn load_config_json(project_dir: &str) -> PyResult { - compiler_guard(|| { - load::load_config_json(std::path::Path::new(project_dir)).map_err(value_error) - }) -} - #[pyfunction] fn catalogue_json() -> PyResult { compiler_guard(|| { @@ -43,7 +35,6 @@ fn selected_codes_json(request_json: &str) -> PyResult { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; - module.add_function(wrap_pyfunction!(load_config_json, module)?)?; module.add_function(wrap_pyfunction!(catalogue_json, module)?)?; module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?; crate::bindings::_helpers::rules::rows::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 7924011a1a..9daefe42ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -8,4 +8,5 @@ pub(crate) mod relation_keys; pub(crate) mod results; pub(crate) mod rows; pub(crate) mod rule_cache; +pub(crate) mod rules_config; pub(crate) mod skills; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 23a5478684..0ac03ac4cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, results, rule_cache, - skills, + rules_config, skills, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -14,6 +14,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { custom_rule_evidence::register(module)?; rule_cache::register(module)?; custom_rule_cache::register(module)?; + rules_config::register(module)?; results::register(module)?; skills::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs new file mode 100644 index 0000000000..436ffbffa2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -0,0 +1,263 @@ +//! Rules configuration loaded natively and returned as the Python configuration models. + +use std::path::PathBuf; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, Py, pyfunction, wrap_pyfunction}; +use serde_json::Value; +use sqlbuild_rules::configuration::main::forbidden_exceptions::forbidden_exceptions; +use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config as load_config; +use sqlbuild_rules::models::RulesConfig; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; + +struct Models<'py> { + module: Bound<'py, PyModule>, +} + +impl<'py> Models<'py> { + fn build( + &self, + class: &str, + fields: &[(&str, Bound<'py, PyAny>)], + ) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(self.module.py()); + for (name, value) in fields { + arguments.set_item(name, value)?; + } + self.module.getattr(class)?.call((), Some(&arguments)) + } +} + +fn strings<'py>(py: Python<'py>, values: &[String]) -> PyResult> { + Ok(PyTuple::new(py, values)?.into_any()) +} + +fn text<'py>(py: Python<'py>, value: &str) -> PyResult> { + value.into_bound_py_any(py) +} + +fn integers<'py>( + py: Python<'py>, + values: &std::collections::BTreeMap, +) -> PyResult> { + values.into_bound_py_any(py) +} + +/// A scalar option as Python reads it from JSON; floats, tables and nulls are dropped. +fn option_scalar<'py>(py: Python<'py>, value: &Value) -> PyResult>> { + match value { + Value::Bool(flag) => Ok(Some(flag.into_bound_py_any(py)?)), + Value::String(item) => Ok(Some(item.into_bound_py_any(py)?)), + Value::Number(number) => match (number.as_i64(), number.as_u64()) { + (Some(integer), _) => Ok(Some(integer.into_bound_py_any(py)?)), + (None, Some(integer)) => Ok(Some(integer.into_bound_py_any(py)?)), + (None, None) => Ok(None), + }, + _ => Ok(None), + } +} + +/// A list option item as JSON decoding yields it. +fn option_item<'py>(py: Python<'py>, value: &Value) -> PyResult> { + match value { + Value::Array(items) => { + let values: Vec> = items + .iter() + .map(|item| option_item(py, item)) + .collect::>()?; + Ok(pyo3::types::PyList::new(py, values)?.into_any()) + } + Value::Object(entries) => { + let table: Bound<'py, PyDict> = PyDict::new(py); + for (key, item) in entries { + table.set_item(key, option_item(py, item)?)?; + } + Ok(table.into_any()) + } + Value::Null => Ok(py.None().into_bound(py)), + Value::Number(number) if number.as_i64().is_none() && number.as_u64().is_none() => { + number.as_f64().unwrap_or(f64::NAN).into_bound_py_any(py) + } + other => Ok(option_scalar(py, other)?.unwrap_or_else(|| py.None().into_bound(py))), + } +} + +fn rule_options<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> { + let result: Bound<'py, PyDict> = PyDict::new(py); + for (code, options) in &config.rule_options { + let values: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in options { + if let Value::Array(items) = value { + let tuple: Vec> = items + .iter() + .map(|item| option_item(py, item)) + .collect::>()?; + values.set_item(name, PyTuple::new(py, tuple)?)?; + } else if let Some(scalar) = option_scalar(py, value)? { + values.set_item(name, scalar)?; + } + } + result.set_item(code, values)?; + } + Ok(result.into_any()) +} + +fn tables<'py, T>( + py: Python<'py>, + items: &[T], + build: impl Fn(&T) -> PyResult>, +) -> PyResult> { + let values: Vec> = items.iter().map(build).collect::>()?; + Ok(PyTuple::new(py, values)?.into_any()) +} + +fn config_object<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> { + let models = Models { + module: py.import(MODELS_MODULE)?, + }; + let threshold_overrides = tables(py, &config.threshold_overrides, |item| { + models.build( + "ThresholdOverride", + &[ + ("paths", strings(py, &item.paths)?), + ("thresholds", integers(py, &item.thresholds)?), + ("reason", text(py, &item.reason)?), + ], + ) + })?; + let rule_exceptions = tables(py, &config.rule_exceptions, |item| { + models.build( + "RuleExemption", + &[ + ("rule", text(py, &item.rule)?), + ("path", text(py, &item.path)?), + ("reason", text(py, &item.reason)?), + ], + ) + })?; + let graph_edge_exceptions = tables(py, &config.graph_edge_exceptions, |item| { + models.build( + "GraphEdgeExclusion", + &[ + ("consumer", text(py, &item.consumer)?), + ("dependency", text(py, &item.dependency)?), + ("reason", text(py, &item.reason)?), + ], + ) + })?; + let rule_ignores = tables(py, &config.rule_ignores, |item| { + models.build( + "RuleIgnore", + &[ + ("rules", strings(py, &item.rules)?), + ("paths", strings(py, &item.paths)?), + ("reason", text(py, &item.reason)?), + ("selectors", strings(py, &item.selectors)?), + ], + ) + })?; + let select_star_allow = tables(py, &config.select_star_allow, |item| { + models.build( + "SelectStarAllow", + &[ + ("paths", strings(py, &item.paths)?), + ("reason", text(py, &item.reason)?), + ], + ) + })?; + let mut layout: Vec<(&str, Bound<'py, PyAny>)> = + vec![("domain_roots", strings(py, &config.layout.domain_roots)?)]; + if !config.layout.levels.is_empty() { + layout.push(("levels", strings(py, &config.layout.levels)?)); + } + models.build( + "RulesConfig", + &[ + ("select", strings(py, &config.select)?), + ("ignore", strings(py, &config.ignore)?), + ("thresholds", integers(py, &config.thresholds)?), + ("threshold_overrides", threshold_overrides), + ("rule_options", rule_options(py, config)?), + ("rule_exceptions", rule_exceptions), + ("graph_edge_exceptions", graph_edge_exceptions), + ("rule_ignores", rule_ignores), + ( + "allow_exceptions", + config.allow_exceptions.into_bound_py_any(py)?, + ), + ("select_star_allow", select_star_allow), + ("domains", strings(py, &config.domains)?), + ( + "approved_source_tokens", + strings(py, &config.approved_source_tokens)?, + ), + ( + "retired_source_tokens", + config.retired_source_tokens.clone().into_bound_py_any(py)?, + ), + ( + "sql_tests", + models.build( + "SqlTestRulesConfig", + &[( + "pipeline_directory", + text(py, &config.sql_tests.pipeline_directory)?, + )], + )?, + ), + ("layout", models.build("LayoutConfig", &layout)?), + ( + "cache", + models.build( + "RulesCacheConfig", + &[("enabled", config.cache.enabled.into_bound_py_any(py)?)], + )?, + ), + ], + ) +} + +fn rules_error(py: Python<'_>, class: &str, message: String, help: Option) -> PyErr { + let raised = || -> PyResult { + let error_class: Bound<'_, PyAny> = py.import(ERRORS_MODULE)?.getattr(class)?; + let error: Bound<'_, PyAny> = match help { + Some(help) => { + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("help", help)?; + error_class.call((message,), Some(&arguments))? + } + None => error_class.call1((message,))?, + }; + Ok(PyErr::from_value(error)) + }; + raised().unwrap_or_else(|error| error) +} + +/// The project's validated rules configuration as `RulesConfig`. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +fn load_rules_config(py: Python<'_>, project_dir: PathBuf) -> PyResult> { + let config: RulesConfig = compiler_guard(|| Ok(load_config(&project_dir)))? + .map_err(|message| rules_error(py, "RulesError", message, None))?; + if let Some(error) = forbidden_exceptions(&config) { + return Err(rules_error( + py, + "RulesConfigError", + error.message, + Some(error.help), + )); + } + Ok(config_object(py, &config)?.unbind()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(load_rules_config, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index f36c31c00f..a7d43d7096 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -2,11 +2,6 @@ use crate::models::RulesConfig; use fensu_policy::policy::types::RuleCodeGrammar; use std::path::Path; -pub(crate) fn load_config_json(project_dir: &Path) -> Result { - let config = load(project_dir)?; - serde_json::to_string(&config).map_err(|error| error.to_string()) -} - pub(crate) fn load(project_dir: &Path) -> Result { let path = project_dir.join("sqlbuild_project.toml"); let source = std::fs::read_to_string(&path).map_err(|error| { diff --git a/crates/sqlbuild-rules/src/configuration/errors.rs b/crates/sqlbuild-rules/src/configuration/errors.rs new file mode 100644 index 0000000000..cd92448b71 --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/errors.rs @@ -0,0 +1,8 @@ +//! Rules configuration that a project's own policy rejects. + +/// A configuration policy violation, with help showing the setting to change. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConfigPolicyError { + pub message: String, + pub help: String, +} diff --git a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs new file mode 100644 index 0000000000..ed049fb702 --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs @@ -0,0 +1,57 @@ +//! `forbidden_exceptions`: exceptions or scoped ignores in a project that disallows them. + +use sqlbuild_analysis::contracts::main::join_helps::join_helps; +use sqlbuild_core::cli_text::main::format_count_noun::format_count_noun; + +use crate::configuration::main::setting_help::setting_help; +use crate::configuration::errors::ConfigPolicyError; +use crate::constants::PROJECT_CONFIG_FILE; +use crate::models::RulesConfig; + +/// The policy error when `allow_exceptions = false` and exception or ignore entries exist. +#[must_use] +pub fn forbidden_exceptions(config: &RulesConfig) -> Option { + if config.allow_exceptions { + return None; + } + let mut entries: Vec = Vec::new(); + let mut tables: Vec = Vec::new(); + for (key, count) in [ + ("rule_exceptions", config.rule_exceptions.len()), + ("rule_ignores", config.rule_ignores.len()), + ] { + if count == 0 { + continue; + } + let table: String = format!("[[rules.{key}]]"); + let count: i64 = i64::try_from(count).unwrap_or(i64::MAX); + entries.push(format_count_noun( + count, + &format!("{table} entry"), + Some(&format!("{table} entries")), + )); + tables.push(table); + } + if entries.is_empty() { + return None; + } + let message: String = format!( + "{PROJECT_CONFIG_FILE} has {}, which this project does not allow; \ + {PROJECT_CONFIG_FILE} sets [rules] allow_exceptions = false", + entries.join(" and ") + ); + let help: String = join_helps(&[ + format!( + "remove every {} entry and fix the findings it hides; to turn a Rule off for the \ + whole project, add its code to [rules] ignore", + tables.join(" and ") + ), + setting_help( + "to allow Rule exceptions, scoped ignores and inline suppressions", + "rules", + "allow_exceptions", + "true", + ), + ]); + Some(ConfigPolicyError { message, help }) +} diff --git a/crates/sqlbuild-rules/src/configuration/main/load.rs b/crates/sqlbuild-rules/src/configuration/main/load.rs deleted file mode 100644 index 43ca577d63..0000000000 --- a/crates/sqlbuild-rules/src/configuration/main/load.rs +++ /dev/null @@ -1,5 +0,0 @@ -use std::path::Path; - -pub fn load_config_json(project_dir: &Path) -> Result { - crate::configuration::_helpers::loading::load_config_json(project_dir) -} diff --git a/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs b/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs new file mode 100644 index 0000000000..2c3ba017d0 --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs @@ -0,0 +1,10 @@ +//! `load_rules_config`: the validated `[rules]` configuration of one project. + +use std::path::Path; + +use crate::models::RulesConfig; + +/// The project's rules configuration, or the reason it cannot be loaded. +pub fn load_rules_config(project_dir: &Path) -> Result { + crate::configuration::_helpers::loading::load(project_dir) +} diff --git a/crates/sqlbuild-rules/src/configuration/main/mod.rs b/crates/sqlbuild-rules/src/configuration/main/mod.rs index fbd112cce3..42488f2f4f 100644 --- a/crates/sqlbuild-rules/src/configuration/main/mod.rs +++ b/crates/sqlbuild-rules/src/configuration/main/mod.rs @@ -1,4 +1,5 @@ -pub mod load; +pub mod forbidden_exceptions; +pub mod load_rules_config; pub mod setting_help; pub mod toml_string_array; pub mod validate; diff --git a/crates/sqlbuild-rules/src/configuration/mod.rs b/crates/sqlbuild-rules/src/configuration/mod.rs index 0b794b2977..1332d0d6b3 100644 --- a/crates/sqlbuild-rules/src/configuration/mod.rs +++ b/crates/sqlbuild-rules/src/configuration/mod.rs @@ -1,2 +1,3 @@ pub mod _helpers; pub mod main; +pub mod errors; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d29993b5bf..773bfedd25 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1772,7 +1772,7 @@ def prepare_lint_sql_batch( requests: list[LintPreparationRequest], ) -> list[tuple[bool, tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None]]: ... def lint_backtick_identifiers(dialect: str) -> bool: ... -def load_config_json(project_dir: str) -> str: ... +def load_rules_config(*, project_dir: Path) -> RulesConfig: ... def selected_codes_json(request_json: str) -> str: ... def render_owned_skill(content: str, input_fingerprint: str) -> str: ... def skill_freshness(content: str | None, input_fingerprint: str) -> str: ... diff --git a/src/sqlbuild/rule_engine/_helpers/engine/config.py b/src/sqlbuild/rule_engine/_helpers/engine/config.py index 383f54139b..17c5eea811 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/config.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/config.py @@ -1,139 +1,16 @@ -"""Native-owned strict rules configuration loading.""" +"""Resolve rule ignore selectors against the compiled project.""" from __future__ import annotations from dataclasses import replace from pathlib import Path -from typing import cast -from sqlbuild._native import format_count_noun from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME from sqlbuild.compiler.planner.main.selection.scope import build_planner_scope from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.errors.setting_help.main.join_helps import join_helps -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.errors.setting_help.main.setting_note import setting_note -from sqlbuild.rule_engine._helpers.engine.native import load_native_config -from sqlbuild.rule_engine.constants import ( - ALLOW_EXCEPTIONS_KEY, - RULE_EXCEPTIONS_KEY, - RULE_IGNORES_KEY, - RULES_CONFIG_SECTION, -) -from sqlbuild.rule_engine.exceptions import RulesConfigError, RulesError -from sqlbuild.rule_engine.models import ( - GraphEdgeExclusion, - LayoutConfig, - RuleExemption, - RuleIgnore, - RulesCacheConfig, - RulesConfig, - SelectStarAllow, - SqlTestRulesConfig, - ThresholdOverride, -) -from sqlbuild.rule_engine.types import RuleOptionValue - - -def load_rules_config(project_dir: Path) -> RulesConfig: - """Load configuration validated and normalized by the native engine.""" - - payload: dict[str, object] = load_native_config(project_dir) - config: RulesConfig = RulesConfig( - select=_strings(payload.get("select")), - ignore=_strings(payload.get("ignore")), - thresholds=_integers(payload.get("thresholds")), - threshold_overrides=tuple( - ThresholdOverride( - paths=_strings(item.get("paths")), - thresholds=_integers(item.get("thresholds")), - reason=str(item["reason"]), - ) - for item in _tables(payload.get("threshold_overrides")) - ), - rule_options=_rule_options(payload.get("rule_options")), - rule_exceptions=tuple( - RuleExemption( - rule=str(item["rule"]), path=str(item["path"]), reason=str(item["reason"]) - ) - for item in _tables(payload.get("rule_exceptions")) - ), - graph_edge_exceptions=tuple( - GraphEdgeExclusion( - consumer=str(item["consumer"]), - dependency=str(item["dependency"]), - reason=str(item["reason"]), - ) - for item in _tables(payload.get("graph_edge_exceptions")) - ), - rule_ignores=tuple( - RuleIgnore( - rules=_strings(item.get("rules")), - paths=_strings(item.get("paths")), - reason=str(item["reason"]), - selectors=_strings(item.get("selectors")), - ) - for item in _tables(payload.get("rule_ignores")) - ), - allow_exceptions=payload.get("allow_exceptions") is not False, - select_star_allow=tuple( - SelectStarAllow(paths=_strings(item.get("paths")), reason=str(item["reason"])) - for item in _tables(payload.get("select_star_allow")) - ), - domains=_strings(payload.get("domains")), - approved_source_tokens=_strings(payload.get("approved_source_tokens")), - retired_source_tokens=_string_mapping(payload.get("retired_source_tokens")), - sql_tests=_sql_tests(payload.get("sql_tests")), - layout=_layout(payload.get("layout")), - cache=_cache(payload.get("cache")), - ) - _reject_forbidden_exceptions(config) - return config - - -def _reject_forbidden_exceptions(config: RulesConfig) -> None: - """Fail when `[rules] allow_exceptions = false` and exceptions or scoped ignores exist.""" - - if config.allow_exceptions: - return - entries: list[str] = [] - tables: list[str] = [] - for key, count in ( - (RULE_EXCEPTIONS_KEY, len(config.rule_exceptions)), - (RULE_IGNORES_KEY, len(config.rule_ignores)), - ): - if count: - table: str = f"[[{RULES_CONFIG_SECTION}.{key}]]" - tables.append(table) - entries.append( - format_count_noun(count=count, singular=f"{table} entry", plural=f"{table} entries") - ) - if not entries: - return - raise RulesConfigError( - f"{PROJECT_CONFIG_FILENAME} has {' and '.join(entries)}, which this project does not " - "allow; " - + setting_note( - file_name=PROJECT_CONFIG_FILENAME, - section=RULES_CONFIG_SECTION, - key=ALLOW_EXCEPTIONS_KEY, - value=False, - ), - help=join_helps( - f"remove every {' and '.join(tables)} entry and fix the findings it hides; " - f"to turn a Rule off for the whole project, add its code to " - f"[{RULES_CONFIG_SECTION}] ignore", - setting_help( - purpose="to allow Rule exceptions, scoped ignores and inline suppressions", - file_name=PROJECT_CONFIG_FILENAME, - section=RULES_CONFIG_SECTION, - key=ALLOW_EXCEPTIONS_KEY, - value=True, - ), - ), - ) +from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild.rule_engine.models import RuleIgnore, RulesConfig def resolve_rule_ignore_selectors( @@ -173,71 +50,3 @@ def _resource_paths(*, project: CompiledProjectView) -> dict[CompiledObjectKey, paths.update({seed.key: seed.seed_file.relative_path for seed in project.seeds}) paths.update({source.key: source.source_file.relative_path for source in project.sources}) return paths - - -def _strings(value: object) -> tuple[str, ...]: - return tuple(str(item) for item in value) if isinstance(value, list) else () - - -def _tables(value: object) -> tuple[dict[str, object], ...]: - if not isinstance(value, list): - return () - tables: list[dict[str, object]] = [] - for table in value: - if isinstance(table, dict): - tables.append({str(key): item for key, item in table.items()}) - return tuple(tables) - - -def _integers(value: object) -> dict[str, int]: - if not isinstance(value, dict): - return {} - return {str(key): item for key, item in value.items() if isinstance(item, int)} - - -def _string_mapping(value: object) -> dict[str, str]: - if not isinstance(value, dict): - return {} - return {str(key): str(item) for key, item in value.items()} - - -def _rule_options(value: object) -> dict[str, dict[str, RuleOptionValue]]: - if not isinstance(value, dict): - return {} - result: dict[str, dict[str, RuleOptionValue]] = {} - for code, raw_options in value.items(): - if not isinstance(raw_options, dict): - continue - options: dict[str, RuleOptionValue] = {} - for name, item in raw_options.items(): - if isinstance(item, list): - options[str(name)] = cast("RuleOptionValue", tuple(item)) - elif isinstance(item, (bool, int, str)): - options[str(name)] = item - result[str(code)] = options - return result - - -def _cache(value: object) -> RulesCacheConfig: - table: dict[str, object] = ( - {str(key): item for key, item in value.items()} if isinstance(value, dict) else {} - ) - return RulesCacheConfig(enabled=bool(table.get("enabled", True))) - - -def _sql_tests(value: object) -> SqlTestRulesConfig: - table: dict[str, object] = ( - {str(key): item for key, item in value.items()} if isinstance(value, dict) else {} - ) - return SqlTestRulesConfig(pipeline_directory=str(table.get("pipeline_directory", "pipelines"))) - - -def _layout(value: object) -> LayoutConfig: - table: dict[str, object] = ( - {str(key): item for key, item in value.items()} if isinstance(value, dict) else {} - ) - levels: tuple[str, ...] = _strings(table.get("levels")) - return LayoutConfig( - levels=levels or LayoutConfig().levels, - domain_roots=_strings(table.get("domain_roots")), - ) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index d50dc4b254..9b22ab8bde 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -229,18 +229,6 @@ def _finding_identity(finding: Finding) -> tuple[str, str, int, int, str]: ) -def load_native_config(project_dir: Path) -> dict[str, object]: - """Load strict rules TOML through the native configuration owner.""" - - try: - payload: object = json.loads(_native.load_config_json(str(project_dir.resolve()))) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - if not isinstance(payload, dict): - raise RulesError("native rules engine returned invalid configuration") - return {str(key): value for key, value in payload.items()} - - def native_catalogue() -> tuple[dict[str, object], ...]: """Return native-owned built-in rule metadata.""" diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index 7096622f1d..2d039faef4 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -10,7 +10,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.rule_engine._helpers.engine.config import load_rules_config from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, @@ -18,6 +17,7 @@ from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.exceptions import RuleAssertionError, RuleUsageError from sqlbuild.rule_engine.main._evaluate import evaluate +from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import ( Rule, RuleCase, @@ -39,7 +39,7 @@ def run_rule_case(*, rule: RuleCheck | Rule, test_case: RuleCase) -> RuleResult: root: Path = Path(temporary) _ = _write_rule_project(root=root, rule=resolved, test_case=test_case) project: CompiledProjectView = _compile_rule_project(root=root) - config: RulesConfig = load_rules_config(root) + config: RulesConfig = load_rules_config(project_dir=root) result: RulesResult = evaluate( project=project, config=config, diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index 49fe4a4a5e..580c8d57a4 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -162,7 +162,3 @@ FACT_DECLARATIONS_PUBLIC_CONSTANTS: str = "declarations.public_constants" FACT_DECLARATIONS_ENUMS: str = "declarations.enums" FACT_DECLARATIONS_CONSTANTS: str = "declarations.constants" -RULES_CONFIG_SECTION: str = "rules" -ALLOW_EXCEPTIONS_KEY: str = "allow_exceptions" -RULE_EXCEPTIONS_KEY: str = "rule_exceptions" -RULE_IGNORES_KEY: str = "rule_ignores" diff --git a/src/sqlbuild/rule_engine/main/load_config.py b/src/sqlbuild/rule_engine/main/load_config.py index c615b541ae..eb3a354929 100644 --- a/src/sqlbuild/rule_engine/main/load_config.py +++ b/src/sqlbuild/rule_engine/main/load_config.py @@ -2,11 +2,11 @@ from pathlib import Path -from sqlbuild.rule_engine._helpers.engine.config import load_rules_config as _load_rules_config +from sqlbuild._native import load_rules_config as _load_rules_config from sqlbuild.rule_engine.models import RulesConfig def load_rules_config(*, project_dir: Path) -> RulesConfig: """Load strict rules configuration from a SQLBuild project.""" - return _load_rules_config(project_dir) + return _load_rules_config(project_dir=project_dir.resolve()) From db02f94abf00d4f02b8f73c7f198c2ae81059bf6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:11:06 +0100 Subject: [PATCH 29/37] refactor(rules): convert rule findings and SQL test rows natively The rules request binding now reads compiled SQL tests, scenarios and initial findings straight from the Python objects, and evaluation, finalisation, the SQL rule cache lookup and the custom-rule cache return Finding models built natively. Finalisation keeps each original finding's fixable mark natively, and undecodable cached SQL findings are dropped as misses inside the lookup, so its handled-error entry is removed. The Python row builders and finding decoders are deleted; a new NativeRulesRequest.sql_facts_json accessor lets tests inspect the facts sent to the engine. --- .../_helpers/rules/custom_rule_cache.rs | 44 +-- .../src/bindings/_helpers/rules/findings.rs | 105 ++++++ .../src/bindings/_helpers/rules/mod.rs | 2 + .../src/bindings/_helpers/rules/rows.rs | 312 +++++------------- .../src/bindings/_helpers/rules/rule_cache.rs | 30 +- .../_helpers/rules/sql_test_objects.rs | 216 ++++++++++++ .../main/forbidden_exceptions.rs | 2 +- .../sqlbuild-rules/src/configuration/mod.rs | 2 +- python-boundary.toml | 4 - src/sqlbuild/_native.pyi | 18 +- .../rule_engine/_helpers/engine/native.py | 82 +---- .../rule_engine/_helpers/run/native_rows.py | 122 +------ .../rule_engine/_helpers/run/rules.py | 12 +- src/sqlbuild/rule_engine/models.py | 4 +- src/sqlbuild/rule_engine/types.py | 1 - .../rule_engine/_helpers/engine/helpers.py | 63 +++- .../_helpers/engine/test_custom_rule_reads.py | 4 +- .../_helpers/engine/test_native_facts.py | 10 +- 18 files changed, 534 insertions(+), 499 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs index 07364fc1e3..8f8e0e3485 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -4,9 +4,8 @@ use std::collections::BTreeMap; use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use pyo3::{IntoPyObjectExt, Py, pyclass, pyfunction, pymethods, wrap_pyfunction}; -use serde_json::Value; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_rules::custom_rule_cache::main::custom_rule_cache_text::custom_rule_cache_text; use sqlbuild_rules::custom_rule_cache::main::lookup_custom_rule_cache::lookup_custom_rule_cache; use sqlbuild_rules::custom_rule_cache::main::ordered_custom_findings::ordered_custom_findings; @@ -24,6 +23,7 @@ use sqlbuild_rules::custom_rule_cache::types::{ReadSet, StoredRules, SubjectResu use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::rules::findings::{FindingClasses, invalid_finding}; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const RULES_ERROR: &str = "RulesError"; @@ -57,33 +57,6 @@ fn custom_rule_identity(request: Option<&Bound<'_, PyDict>>) -> PyResult })) } -fn python_value(py: Python<'_>, value: &Value) -> PyResult> { - match value { - Value::Null => Ok(py.None()), - Value::Bool(flag) => flag.into_py_any(py), - Value::Number(number) => match (number.as_i64(), number.as_u64()) { - (Some(integer), _) => integer.into_py_any(py), - (None, Some(integer)) => integer.into_py_any(py), - (None, None) => number.as_f64().unwrap_or(f64::NAN).into_py_any(py), - }, - Value::String(text) => text.into_py_any(py), - Value::Array(items) => { - let values: Vec> = items - .iter() - .map(|item| python_value(py, item)) - .collect::>()?; - PyList::new(py, values)?.into_py_any(py) - } - Value::Object(entries) => { - let dictionary: Bound<'_, PyDict> = PyDict::new(py); - for (key, item) in entries { - dictionary.set_item(key, python_value(py, item)?)?; - } - dictionary.into_py_any(py) - } - } -} - fn readset_tuple<'py>(py: Python<'py>, readset: &ReadSet) -> PyResult> { let keys: Vec> = readset .iter() @@ -232,11 +205,16 @@ impl CustomRuleCache { Ok(()) } - /// Every finding payload in rule code, then subject order. - fn findings(&self, py: Python<'_>) -> PyResult>> { + /// Every finding in rule code, then subject order; a malformed payload is a `RulesError`. + fn findings<'py>(&self, py: Python<'py>) -> PyResult>> { + let classes = FindingClasses::new(py)?; ordered_custom_findings(&self.rules, &self.results) .iter() - .map(|finding| python_value(py, finding)) + .map(|finding| { + classes + .finding_of_payload(finding)? + .ok_or_else(|| invalid_finding(py)) + }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs new file mode 100644 index 0000000000..93fd0e3775 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs @@ -0,0 +1,105 @@ +//! Rule findings converted between the Python `Finding` model and native rows and payloads. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::Value; +use sqlbuild_rules::models::Fault; + +const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const FINDING_CLASS: &str = "Finding"; +const PATHLIB_MODULE: &str = "pathlib"; +const PATH_CLASS: &str = "Path"; +const ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERROR: &str = "RulesError"; +const INVALID_FINDING: &str = "native rules engine returned an invalid finding"; + +/// The Python `Finding` and `Path` classes, imported once per conversion batch. +pub(crate) struct FindingClasses<'py> { + finding: Bound<'py, PyAny>, + path: Bound<'py, PyAny>, +} + +impl<'py> FindingClasses<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + Ok(Self { + finding: py.import(MODELS_MODULE)?.getattr(FINDING_CLASS)?, + path: py.import(PATHLIB_MODULE)?.getattr(PATH_CLASS)?, + }) + } + + /// One finding from its native fault row, optionally marked fixable. + pub(crate) fn finding_of_fault( + &self, + fault: Fault, + fixable: bool, + ) -> PyResult> { + let py: Python<'py> = self.finding.py(); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("code", fault.code)?; + arguments.set_item("path", self.path.call1((fault.path,))?)?; + arguments.set_item("line", fault.line)?; + arguments.set_item("column", fault.column)?; + arguments.set_item("message", fault.message)?; + arguments.set_item("remediation", fault.remediation)?; + arguments.set_item("unevaluated", fault.unevaluated)?; + if fixable { + arguments.set_item("fixable", true)?; + } + self.finding.call((), Some(&arguments)) + } + + /// One finding from a stored or custom-rule payload, or `None` when the payload is malformed. + pub(crate) fn finding_of_payload(&self, value: &Value) -> PyResult>> { + let Some(payload) = value.as_object() else { + return Ok(None); + }; + let text = |key: &str| payload.get(key).and_then(Value::as_str).map(str::to_owned); + let number = |key: &str| payload.get(key).and_then(Value::as_u64); + let flag = |key: &str| payload.get(key).and_then(Value::as_bool) == Some(true); + let (Some(code), Some(path), Some(line), Some(column), Some(message), Some(remediation)) = ( + text("code"), + text("path"), + number("line"), + number("column"), + text("message"), + text("remediation"), + ) else { + return Ok(None); + }; + let fault = Fault { + unevaluated: flag("unevaluated"), + code, + path, + line, + column, + message, + remediation, + }; + Ok(Some(self.finding_of_fault(fault, flag("fixable"))?)) + } +} + +/// The native fault row of one Python `Finding`. +pub(crate) fn finding_fault(finding: &Bound<'_, PyAny>) -> PyResult { + Ok(Fault { + unevaluated: finding.getattr("unevaluated")?.extract()?, + code: finding.getattr("code")?.extract()?, + path: finding + .getattr("path")? + .call_method0("as_posix")? + .extract()?, + line: finding.getattr("line")?.extract()?, + column: finding.getattr("column")?.extract()?, + message: finding.getattr("message")?.extract()?, + remediation: finding.getattr("remediation")?.extract()?, + }) +} + +/// The `RulesError` a malformed native or custom-rule finding raises. +pub(crate) fn invalid_finding(py: Python<'_>) -> PyErr { + let raised = || -> PyResult { + let class: Bound<'_, PyAny> = py.import(ERRORS_MODULE)?.getattr(RULES_ERROR)?; + Ok(PyErr::from_value(class.call1((INVALID_FINDING,))?)) + }; + raised().unwrap_or_else(|error| error) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 9daefe42ba..e0119dfd93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod custom_rule_cache; pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; pub(crate) mod evaluation; +pub(crate) mod findings; pub(crate) mod lint; pub(crate) mod registration; pub(crate) mod relation_keys; @@ -10,3 +11,4 @@ pub(crate) mod rows; pub(crate) mod rule_cache; pub(crate) mod rules_config; pub(crate) mod skills; +pub(crate) mod sql_test_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 066080368b..54a905591f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -1,40 +1,44 @@ //! Build and evaluate the built-in rules request natively from compiled-project rows. -use std::collections::HashMap; +use std::collections::HashSet; use std::path::PathBuf; use std::sync::Mutex; -use pyo3::exceptions::{PyOSError, PyValueError}; +use pyo3::exceptions::PyOSError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, }; use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods}; -use pyo3::{FromPyObject, pyclass, pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_rules::engine::main::build_models::build_models; use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows; use sqlbuild_rules::engine::main::finalize_rows::finalize_rows; use sqlbuild_rules::engine::main::project_model_rows::project_model_rows; use sqlbuild_rules::errors::RowsError; use sqlbuild_rules::models::{ - Declaration, DeclarationMember, DirectTestResourceKind, EvaluateRequest, Fault, Model, - ModelRow, ModelRowsContext, RowsEvaluation, RowsRequest, RulesConfig, SqlScenarioFact, - SqlTestCteFact, SqlTestFact, SqlTestMode, SqlTestParameterFact, SqlTestParameterValueFact, - TestedResource, + Declaration, DeclarationMember, EvaluateRequest, Fault, Model, ModelRow, ModelRowsContext, + RowsEvaluation, RowsRequest, RulesConfig, }; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; +use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; use crate::bindings::types::CompilerDetach; const LEFT_ARGUMENT: &str = "left"; const RIGHT_ARGUMENT: &str = "right"; -type FaultRow = (bool, String, String, u64, u64, String, String); -type CteRow = (String, String); -type ModelTestPayloadPy = (Vec, Vec, Vec, bool, bool); -type TestNamesRow = (Vec, Vec, Vec, Vec); -type EvaluationRow = (Vec, Vec, u64, u64, u64, u64, bool); +type EvaluationRow<'py> = ( + Vec>, + Vec, + u64, + u64, + u64, + u64, + bool, +); /// Name, relative path, members, value, value type and rendering of one declaration. #[derive(FromPyObject)] @@ -47,42 +51,6 @@ struct DeclarationRowPy<'py>( Option, ); -#[derive(FromPyObject)] -struct SqlTestRowPy<'py>( - String, - String, - u64, - String, - Option, - String, - TestNamesRow, - Vec<(String, String)>, - Option, - Option>, -); - -#[derive(FromPyObject)] -struct SqlTestCaseRowPy<'py>( - Option, - String, - Option, - Option, - Vec<(String, String, bool)>, - Vec<(String, Bound<'py, PyAny>)>, -); - -#[derive(FromPyObject)] -struct ScenarioRowPy( - String, - String, - String, - Option, - Vec, - Vec, - Vec, - Vec, -); - /// The request's project facts, which models, tests and scenarios are built beside. #[derive(FromPyObject)] #[pyo3(from_item_all)] @@ -94,7 +62,7 @@ struct ProjectRowsPy<'py> { attached_audit_targets: Vec, public_enums: Vec>, public_constants: Vec>, - initial_findings: Vec, + initial_findings: Vec>, types_equal: Bound<'py, PyAny>, } @@ -104,27 +72,50 @@ struct NativeRulesRequest { request: Mutex>, } +#[pymethods] +impl NativeRulesRequest { + /// The SQL test and scenario facts this request carries, as `{"sql_tests", "sql_scenarios"}` JSON. + fn sql_facts_json(&self) -> PyResult { + let guard = self + .request + .lock() + .map_err(|_| value_error("rules request lock is poisoned"))?; + let rows: &RowsRequest = guard + .as_ref() + .ok_or_else(|| value_error("rules request was already evaluated"))?; + serde_json::to_string(&serde_json::json!({ + "sql_tests": rows.request.sql_tests, + "sql_scenarios": rows.request.sql_scenarios, + })) + .map_err(value_error) + } +} + #[pyfunction] fn build_rules_request<'py>( mut project: ProjectRowsPy<'py>, models: (PyRef<'py, NativeCompiledProject>, Vec), - sql_tests: Vec>, - scenarios: Vec, + sql_tests: Vec>, + scenarios: Vec>, ) -> PyResult { compiler_guard(|| { let mut request: EvaluateRequest = serde_json::from_slice(project.header_json.as_bytes()).map_err(value_error)?; request.sql_tests = sql_tests - .into_iter() - .map(sql_test) + .iter() + .map(sql_test_fact) + .collect::>()?; + request.sql_scenarios = scenarios + .iter() + .map(scenario_fact) .collect::>()?; - request.sql_scenarios = scenarios.into_iter().map(scenario).collect(); request.public_enums = declarations(std::mem::take(&mut project.public_enums))?; request.public_constants = declarations(std::mem::take(&mut project.public_constants))?; - request.initial_findings = std::mem::take(&mut project.initial_findings) - .into_iter() - .map(fault) - .collect(); + request.initial_findings = project + .initial_findings + .iter() + .map(finding_fault) + .collect::>()?; let rows: Vec = project_model_rows(models.0.facts(), &models.1).map_err(value_error)?; request.models = built_models(&project, &request, rows)?; @@ -139,10 +130,10 @@ fn build_rules_request<'py>( } #[pyfunction] -fn evaluate_rules_request( - py: Python<'_>, - request: &Bound<'_, NativeRulesRequest>, -) -> PyResult { +fn evaluate_rules_request<'py>( + py: Python<'py>, + request: &Bound<'py, NativeRulesRequest>, +) -> PyResult> { let request: RowsRequest = request .borrow() .request @@ -157,8 +148,13 @@ fn evaluate_rules_request( RowsError::Rules(message) => value_error(message), RowsError::Io(error) => PyOSError::new_err(error.to_string()), })?; + let classes = FindingClasses::new(py)?; Ok(( - evaluation.faults.into_iter().map(fault_row).collect(), + evaluation + .faults + .into_iter() + .map(|fault| classes.finding_of_fault(fault, false)) + .collect::>()?, evaluation.selected_codes, evaluation.evaluated_models, evaluation.cache_hits, @@ -175,14 +171,14 @@ struct FinalizeRowsPy<'py> { project_dir: String, config_json: Bound<'py, PyBytes>, evaluated_codes: Vec, - findings: Vec, + findings: Vec>, } #[pyfunction] -fn finalize_rule_findings_rows( - py: Python<'_>, - request: FinalizeRowsPy<'_>, -) -> PyResult> { +fn finalize_rule_findings_rows<'py>( + py: Python<'py>, + request: FinalizeRowsPy<'py>, +) -> PyResult>> { let config: RulesConfig = serde_json::from_slice(request.config_json.as_bytes()) .map_err(|error| value_error(format!("invalid findings request: {error}")))?; let FinalizeRowsPy { @@ -191,11 +187,26 @@ fn finalize_rule_findings_rows( findings, .. } = request; - let faults: Vec = findings.into_iter().map(fault).collect(); + let mut fixable: HashSet<(String, String, u64, u64, String)> = HashSet::new(); + let mut faults: Vec = Vec::with_capacity(findings.len()); + for finding in &findings { + let fault: Fault = finding_fault(finding)?; + if finding.getattr("fixable")?.extract::()? { + fixable.insert(fault_identity(&fault)); + } + faults.push(fault); + } let finalized: Vec = py .compiler_detach(|| finalize_rows(project_dir, config, &evaluated_codes, faults)) .map_err(value_error)?; - Ok(finalized.into_iter().map(fault_row).collect()) + let classes = FindingClasses::new(py)?; + finalized + .into_iter() + .map(|fault| { + let is_fixable: bool = fixable.contains(&fault_identity(&fault)); + classes.finding_of_fault(fault, is_fixable) + }) + .collect() } /// `left=` and `right=`: the callback is `types_equal`, whose arguments are keyword-only. @@ -265,166 +276,13 @@ fn declaration_member((name, value): (String, Bound<'_, PyAny>)) -> PyResult) -> PyResult { - let SqlTestRowPy( - source_path, - ownership_root, - block_index, - name, - explicit_name, - mode, - (expected_model_names, assertion_names, assertion_target_model_names, target_model_names), - tested_resources, - model_payload, - case, - ) = row; - let (authored_ctes, expected_ctes, assertion_ctes, has_macro_mocks, has_model_query_overrides) = - model_payload.unwrap_or_default(); - let mut fact = SqlTestFact { - source_path, - ownership_root, - block_index, - name, - explicit_name, - parent_name: None, - case_name: None, - case_index: None, - case_fingerprint: None, - parameter_schema: Vec::new(), - parameters: Vec::new(), - mode: test_mode(&mode)?, - expected_model_names, - assertion_names, - assertion_target_model_names, - target_model_names, - tested_resources: tested_resources - .into_iter() - .map(|(kind, name)| { - Ok(TestedResource { - kind: tested_kind(&kind)?, - name, - }) - }) - .collect::>()?, - authored_ctes: ctes(authored_ctes), - expected_ctes: ctes(expected_ctes), - assertion_ctes: ctes(assertion_ctes), - has_macro_mocks, - has_model_query_overrides, - empty_input_only: false, - }; - if let Some(SqlTestCaseRowPy( - parent_name, - case_name, - case_index, - case_fingerprint, - schema, - values, - )) = case - { - let types: HashMap<&str, &str> = schema - .iter() - .map(|(name, value_type, _)| (name.as_str(), value_type.as_str())) - .collect(); - fact.parameters = values - .into_iter() - .map(|(name, value)| { - let value_type: String = types - .get(name.as_str()) - .ok_or_else(|| PyValueError::new_err(format!("unknown test parameter {name}")))? - .to_string(); - Ok(SqlTestParameterValueFact { - name, - value_type, - value: plain_value(&value)?, - }) - }) - .collect::>()?; - fact.parameter_schema = schema - .into_iter() - .map(|(name, value_type, nullable)| SqlTestParameterFact { - name, - value_type, - nullable, - }) - .collect(); - fact.parent_name = parent_name; - fact.case_name = Some(case_name); - fact.case_index = case_index; - fact.case_fingerprint = case_fingerprint; - } - Ok(fact) -} - -fn ctes(rows: Vec) -> Vec { - rows.into_iter() - .map(|(name, sql)| SqlTestCteFact { name, sql }) - .collect() -} - -fn scenario(row: ScenarioRowPy) -> SqlScenarioFact { - let ScenarioRowPy( - source_path, - ownership_root, - name, - description, - expected_model_names, - assertion_names, - assertion_target_model_names, - target_model_names, - ) = row; - SqlScenarioFact { - source_path, - ownership_root, - name, - description, - expected_model_names, - assertion_names, - assertion_target_model_names, - target_model_names, - } -} - -fn test_mode(value: &str) -> PyResult { - match value { - "model" => Ok(SqlTestMode::Model), - "macro" => Ok(SqlTestMode::Macro), - "udf" => Ok(SqlTestMode::Udf), - "table_fn" => Ok(SqlTestMode::TableFn), - _ => Err(value_error(format!("unknown SQL test mode {value}"))), - } -} - -fn tested_kind(value: &str) -> PyResult { - match value { - "macro" => Ok(DirectTestResourceKind::Macro), - "udf" => Ok(DirectTestResourceKind::Udf), - "table_fn" => Ok(DirectTestResourceKind::TableFn), - _ => Err(value_error(format!("unknown tested resource kind {value}"))), - } -} - -fn fault((unevaluated, code, path, line, column, message, remediation): FaultRow) -> Fault { - Fault { - unevaluated, - code, - path, - line, - column, - message, - remediation, - } -} - -fn fault_row(fault: Fault) -> FaultRow { +fn fault_identity(fault: &Fault) -> (String, String, u64, u64, String) { ( - fault.unevaluated, - fault.code, - fault.path, + fault.code.clone(), + fault.path.clone(), fault.line, fault.column, - fault.message, - fault.remediation, + fault.message.clone(), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index cccd043426..6600388df3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -4,7 +4,9 @@ use std::collections::BTreeMap; use std::path::PathBuf; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -12,6 +14,7 @@ use serde_json::{Value, json}; use sqlbuild_rules::rule_cache::main::file_rule_key::file_rule_key; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::rules::findings::FindingClasses; use sqlbuild_rules::rule_cache::main::lint_key::lint_key; use sqlbuild_rules::rule_cache::main::lookup_rule_cache::lookup_rule_cache; use sqlbuild_rules::rule_cache::main::model_rule_key::model_rule_key; @@ -117,20 +120,27 @@ fn cached_entries(entries: &StoredEntries) -> PyResult { .collect() } -/// `(hits, misses)`: stored findings JSON of paths whose key matches, and every other path. +/// The stored findings of each path whose key matches; undecodable stored findings miss. #[pyfunction] #[pyo3(signature = (*, entries, keys))] -fn lookup_sql_rule_cache( +fn lookup_sql_rule_cache<'py>( + py: Python<'py>, entries: StoredEntries, keys: Vec<(String, String)>, -) -> PyResult<(BTreeMap, Vec)> { +) -> PyResult>>> { let lookup: CacheLookup = lookup_rule_cache(&cached_entries(&entries)?, &keys); - let hits: BTreeMap = lookup - .hits - .into_iter() - .map(|(path, findings)| Ok((path, serde_json::to_string(&findings).map_err(invalid)?))) - .collect::>()?; - Ok((hits, lookup.misses)) + let classes = FindingClasses::new(py)?; + let mut hits: BTreeMap>> = BTreeMap::new(); + for (path, findings) in lookup.hits { + let decoded: Option>> = findings + .iter() + .map(|finding| classes.finding_of_payload(finding)) + .collect::>()?; + if let Some(decoded) = decoded { + hits.insert(path, decoded); + } + } + Ok(hits) } fn finding_payload(finding: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs new file mode 100644 index 0000000000..899fc77974 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs @@ -0,0 +1,216 @@ +//! Compiled SQL tests and scenarios read into the native rules request's facts. + +use std::collections::HashMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyString; +use sqlbuild_rules::models::{ + DirectTestResourceKind, SqlScenarioFact, SqlTestCteFact, SqlTestFact, SqlTestMode, + SqlTestParameterFact, SqlTestParameterValueFact, TestedResource, +}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compiled_project::records::plain_typed_value; +use crate::bindings::_helpers::compiled_project::values::plain_value; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const MODEL_PAYLOAD_CLASS: &str = "CompiledModelSqlTestPayload"; +const DESCRIPTION_KEY: &str = "description"; + +fn posix(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("as_posix")?.extract() +} + +/// `primary` when it is truthy, otherwise `fallback`, as Python's `or` picks. +fn either<'py>( + primary: Bound<'py, PyAny>, + fallback: impl FnOnce() -> PyResult>, +) -> PyResult> { + if primary.is_truthy()? { + Ok(primary) + } else { + fallback() + } +} + +fn names(owner: &Bound<'_, PyAny>, attribute: &str) -> PyResult> { + owner.getattr(attribute)?.extract() +} + +fn ctes(payload: &Bound<'_, PyAny>, attribute: &str) -> PyResult> { + payload + .getattr(attribute)? + .try_iter()? + .map(|cte| { + let cte = cte?; + Ok(SqlTestCteFact { + name: cte.getattr("name")?.extract()?, + sql: cte.getattr("sql_body")?.extract()?, + }) + }) + .collect() +} + +fn test_mode(value: &str) -> PyResult { + match value { + "model" => Ok(SqlTestMode::Model), + "macro" => Ok(SqlTestMode::Macro), + "udf" => Ok(SqlTestMode::Udf), + "table_fn" => Ok(SqlTestMode::TableFn), + _ => Err(value_error(format!("unknown SQL test mode {value}"))), + } +} + +fn tested_kind(value: &str) -> PyResult { + match value { + "macro" => Ok(DirectTestResourceKind::Macro), + "udf" => Ok(DirectTestResourceKind::Udf), + "table_fn" => Ok(DirectTestResourceKind::TableFn), + _ => Err(value_error(format!("unknown tested resource kind {value}"))), + } +} + +fn tested_resources(test: &Bound<'_, PyAny>) -> PyResult> { + test.getattr("tested_resources")? + .try_iter()? + .map(|resource| { + let resource = resource?; + let kind: String = resource.getattr("kind")?.getattr("value")?.extract()?; + Ok(TestedResource { + kind: tested_kind(&kind)?, + name: resource.getattr("name")?.extract()?, + }) + }) + .collect() +} + +fn with_payload(test: &Bound<'_, PyAny>, mut fact: SqlTestFact) -> PyResult { + let payload: Bound<'_, PyAny> = test.getattr("payload")?; + let model_payload: Bound<'_, PyAny> = test + .py() + .import(COMPILE_MODELS_MODULE)? + .getattr(MODEL_PAYLOAD_CLASS)?; + if !payload.is_instance(&model_payload)? { + return Ok(fact); + } + fact.authored_ctes = ctes(&payload, "authored_ctes")?; + fact.expected_ctes = ctes(&payload, "expected_ctes")?; + fact.assertion_ctes = ctes(&payload, "assertion_ctes")?; + fact.has_macro_mocks = payload.getattr("macro_mocks")?.is_truthy()?; + fact.has_model_query_overrides = payload.getattr("model_query_overrides")?.is_truthy()?; + Ok(fact) +} + +fn with_case(test: &Bound<'_, PyAny>, mut fact: SqlTestFact) -> PyResult { + let case_name: Option = test.getattr("case_name")?.extract()?; + let Some(case_name) = case_name else { + return Ok(fact); + }; + let mut schema: Vec = Vec::new(); + for parameter in test.getattr("parameter_schema")?.try_iter()? { + let parameter = parameter?; + schema.push(SqlTestParameterFact { + name: parameter.getattr("name")?.extract()?, + value_type: parameter + .getattr("value_type")? + .getattr("value")? + .extract()?, + nullable: parameter.getattr("nullable")?.extract()?, + }); + } + let types: HashMap<&str, &str> = schema + .iter() + .map(|parameter| (parameter.name.as_str(), parameter.value_type.as_str())) + .collect(); + let mut parameters: Vec = Vec::new(); + for item in test.getattr("parameter_values")?.try_iter()? { + let (name, value): (String, Bound<'_, PyAny>) = item?.extract()?; + let value_type: String = types + .get(name.as_str()) + .ok_or_else(|| PyValueError::new_err(format!("unknown test parameter {name}")))? + .to_string(); + parameters.push(SqlTestParameterValueFact { + name, + value_type, + value: plain_value(&plain_typed_value(&value)?)?, + }); + } + fact.parameters = parameters; + fact.parameter_schema = schema; + fact.parent_name = test.getattr("parent_name")?.extract()?; + fact.case_name = Some(case_name); + fact.case_index = test.getattr("case_index")?.extract()?; + fact.case_fingerprint = test.getattr("case_fingerprint")?.extract()?; + Ok(fact) +} + +/// One compiled SQL test as the facts the built-in rules read. +pub(crate) fn sql_test_fact(test: &Bound<'_, PyAny>) -> PyResult { + let test_file: Bound<'_, PyAny> = test.getattr("test_file")?; + let source_path: Bound<'_, PyAny> = either(test.getattr("source_path")?, || { + test_file.getattr("relative_path") + })?; + let ownership_root: Bound<'_, PyAny> = either(test.getattr("ownership_root")?, || { + test_file.getattr("ownership_root") + })?; + let block_index: Bound<'_, PyAny> = either(test.getattr("block_index")?, || { + test.getattr("test_block")?.getattr("test_index") + })?; + let mode: String = test.getattr("mode")?.getattr("value")?.extract()?; + let fact = SqlTestFact { + source_path: posix(&source_path)?, + ownership_root: posix(&ownership_root)?, + block_index: block_index.extract()?, + name: test.getattr("name")?.extract()?, + explicit_name: test.getattr("explicit_name")?.extract()?, + parent_name: None, + case_name: None, + case_index: None, + case_fingerprint: None, + parameter_schema: Vec::new(), + parameters: Vec::new(), + mode: test_mode(&mode)?, + expected_model_names: names(test, "expected_model_names")?, + assertion_names: names(test, "assertion_names")?, + assertion_target_model_names: names(test, "assertion_target_model_names")?, + target_model_names: names(test, "target_model_names")?, + tested_resources: tested_resources(test)?, + authored_ctes: Vec::new(), + expected_ctes: Vec::new(), + assertion_ctes: Vec::new(), + has_macro_mocks: false, + has_model_query_overrides: false, + empty_input_only: false, + }; + with_case(test, with_payload(test, fact)?) +} + +/// One compiled SQL scenario as the facts the built-in rules read. +pub(crate) fn scenario_fact(scenario: &Bound<'_, PyAny>) -> PyResult { + let scenario_file: Bound<'_, PyAny> = scenario.getattr("scenario_file")?; + let source_path: Bound<'_, PyAny> = either(scenario.getattr("source_path")?, || { + scenario_file.getattr("relative_path") + })?; + let ownership_root: Bound<'_, PyAny> = either(scenario.getattr("ownership_root")?, || { + scenario_file.getattr("ownership_root") + })?; + let header: Bound<'_, PyAny> = scenario_file + .getattr("header_values")? + .call_method1("get", (DESCRIPTION_KEY,))?; + let description: Option = if header.is_instance_of::() { + Some(header.extract()?) + } else { + None + }; + Ok(SqlScenarioFact { + source_path: posix(&source_path)?, + ownership_root: posix(&ownership_root)?, + name: scenario.getattr("name")?.extract()?, + description, + expected_model_names: names(scenario, "expected_model_names")?, + assertion_names: names(scenario, "assertion_names")?, + assertion_target_model_names: names(scenario, "assertion_target_model_names")?, + target_model_names: names(scenario, "target_model_names")?, + }) +} diff --git a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs index ed049fb702..7108ee1055 100644 --- a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs +++ b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs @@ -3,8 +3,8 @@ use sqlbuild_analysis::contracts::main::join_helps::join_helps; use sqlbuild_core::cli_text::main::format_count_noun::format_count_noun; -use crate::configuration::main::setting_help::setting_help; use crate::configuration::errors::ConfigPolicyError; +use crate::configuration::main::setting_help::setting_help; use crate::constants::PROJECT_CONFIG_FILE; use crate::models::RulesConfig; diff --git a/crates/sqlbuild-rules/src/configuration/mod.rs b/crates/sqlbuild-rules/src/configuration/mod.rs index 1332d0d6b3..3e943c42e4 100644 --- a/crates/sqlbuild-rules/src/configuration/mod.rs +++ b/crates/sqlbuild-rules/src/configuration/mod.rs @@ -1,3 +1,3 @@ pub mod _helpers; -pub mod main; pub mod errors; +pub mod main; diff --git a/python-boundary.toml b/python-boundary.toml index 633a9bc492..6a00506e2a 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -275,10 +275,6 @@ note = "A test file that no longer parses is left unformatted, as before; no Pyt location = "lint/_helpers/native_sql.py::_native_responses" note = "A native lint failure is reported as a lint error result, not recomputed in Python." -[[handled_native_errors]] -location = "rule_engine/_helpers/run/rules.py::_cached_sql_findings" -note = "An undecodable cached finding is a cache miss; the rules run again natively." - [[handled_native_errors]] location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 773bfedd25..90a8a3f391 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -312,17 +312,17 @@ def catalogue_json() -> str: ... class NativeRulesRequest: """A natively built rules request held until it is evaluated once.""" + def sql_facts_json(self) -> str: ... + def build_rules_request( project: dict[str, object], models: tuple[NativeCompiledProject, list[str]], - sql_tests: list[tuple[object, ...]], - scenarios: list[tuple[object, ...]], + sql_tests: list[CompiledSqlTest], + scenarios: list[CompiledSqlScenario], ) -> NativeRulesRequest: ... def evaluate_rules_request( request: NativeRulesRequest, -) -> tuple[ - list[tuple[bool, str, str, int, int, str, str]], list[str], int, int, int, int, bool -]: ... +) -> tuple[list[Finding], list[str], int, int, int, int, bool]: ... def emit_orjson_report(report: object) -> str: ... def emit_json_report(report: object) -> str: ... @@ -1756,9 +1756,7 @@ class PreparedCompileArtifacts: ) -> WrittenTarget | None: ... def planning_diagnostics(self) -> tuple[CompilerDiagnostic, ...] | None: ... -def finalize_rule_findings_rows( - request: dict[str, object], -) -> list[tuple[bool, str, str, int, int, str, str]]: ... +def finalize_rule_findings_rows(request: dict[str, object]) -> list[Finding]: ... def run_custom_host_json(spec_json: str) -> str: ... def format_sql_json(request_json: str) -> str: ... def format_sql_batch_json(request_json: str) -> str: ... @@ -3167,7 +3165,7 @@ def sql_file_rule_key( def load_sql_rule_cache(*, path: Path) -> dict[str, tuple[str, str]]: ... def lookup_sql_rule_cache( *, entries: dict[str, tuple[str, str]], keys: list[tuple[str, str]] -) -> tuple[dict[str, str], list[str]]: ... +) -> dict[str, list[Finding]]: ... def encode_sql_rule_findings(*, findings: list[Finding]) -> str: ... def write_sql_rule_cache(*, path: Path, entries: dict[str, tuple[str, str]]) -> None: ... def cursor_parse(*, raw: object, cursor_type: str) -> CursorScalar: ... @@ -3259,7 +3257,7 @@ class CustomRuleCache: ] ], ) -> None: ... - def findings(self) -> list[dict[str, object]]: ... + def findings(self) -> list[Finding]: ... @property def hits(self) -> int: ... @property diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 9b22ab8bde..7c5d5c1544 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -6,7 +6,7 @@ import json from collections.abc import Callable from concurrent.futures import Executor, Future, ThreadPoolExecutor -from dataclasses import asdict, replace +from dataclasses import asdict from functools import partial from pathlib import Path @@ -117,7 +117,7 @@ def _evaluate_native_rows( ) findings: tuple[Finding, ...] = ( *native.findings, - *(decode_rule_finding(value) for value in custom.findings), + *custom.findings, ) if not defer_suppressions: findings = finalize_native_findings( @@ -196,36 +196,11 @@ def finalize_native_findings( ) -> tuple[Finding, ...]: """Apply exception policy once to completed SQL, native, and custom findings.""" - return _with_fixable( - original=findings, - finalized=finalize_rules_rows( - config=config, - project_dir=project_dir, - evaluated_codes=evaluated_codes, - findings=findings, - ), - ) - - -def _with_fixable( - *, original: tuple[Finding, ...], finalized: tuple[Finding, ...] -) -> tuple[Finding, ...]: - fixable: frozenset[tuple[str, str, int, int, str]] = frozenset( - _finding_identity(finding) for finding in original if finding.fixable - ) - return tuple( - replace(finding, fixable=True) if _finding_identity(finding) in fixable else finding - for finding in finalized - ) - - -def _finding_identity(finding: Finding) -> tuple[str, str, int, int, str]: - return ( - finding.code, - finding.path.as_posix(), - finding.line, - finding.column, - finding.message, + return finalize_rules_rows( + config=config, + project_dir=project_dir, + evaluated_codes=evaluated_codes, + findings=findings, ) @@ -378,46 +353,3 @@ def _custom_rule_source_payload(*, rule: Rule, project_dir: Path) -> dict[str, o "source_column": 1, "owner": f"{source or 'rules'}:{check_owner}", } - - -def decode_rule_finding(value: object) -> Finding: - if not isinstance(value, dict): - raise RulesError("native rules engine returned an invalid finding") - payload: dict[str, object] = {str(key): item for key, item in value.items()} - code: object = payload.get("code") - path: object = payload.get("path") - line: object = payload.get("line") - column: object = payload.get("column") - message: object = payload.get("message") - remediation: object = payload.get("remediation") - if ( - not isinstance(code, str) - or not isinstance(path, str) - or not isinstance(line, int) - or not isinstance(column, int) - or not isinstance(message, str) - or not isinstance(remediation, str) - ): - raise RulesError("native rules engine returned an invalid finding") - return Finding( - code=code, - path=Path(path), - line=line, - column=column, - message=message, - remediation=remediation, - unevaluated=payload.get("unevaluated") is True, - fixable=payload.get("fixable") is True, - ) - - -def _finding_payload(finding: Finding) -> dict[str, object]: - return { - "unevaluated": finding.unevaluated, - "code": finding.code, - "path": finding.path.as_posix(), - "line": finding.line, - "column": finding.column, - "message": finding.message, - "remediation": finding.remediation, - } diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py index 5f67c8da7e..843d59146f 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py @@ -9,18 +9,8 @@ import orjson import sqlbuild._native as _native -from sqlbuild._native import ( - compiled_project_facts, - declaration_rows, - plain_typed_value, -) +from sqlbuild._native import compiled_project_facts, declaration_rows from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.compile.models import ( - CompiledModelSqlTestPayload, - CompiledSqlScenario, - CompiledSqlTest, - CompileSqlTestCte, -) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection @@ -32,7 +22,6 @@ ) from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import Finding, NativeRulesEvaluation, RulesConfig -from sqlbuild.rule_engine.types import FindingRow def build_rules_request( @@ -85,7 +74,7 @@ def build_rules_request( ], "public_enums": public_enums, "public_constants": public_constants, - "initial_findings": [finding_row(finding) for finding in initial_findings], + "initial_findings": list(initial_findings), "types_equal": partial(types_equal, dialect=dialect), } try: @@ -93,8 +82,8 @@ def build_rules_request( request = _native.build_rules_request( project_rows, (compiled_project_facts(project), [model.name for model in project.models]), - [_sql_test_row(test) for test in project.sql_tests], - [_scenario_row(scenario) for scenario in project.sql_scenarios], + list(project.sql_tests), + list(project.sql_scenarios), ) except (ValueError, TypeError) as error: raise RulesError(str(error)) from error @@ -111,7 +100,7 @@ def evaluate_rules_request(request: _native.NativeRulesRequest) -> NativeRulesEv except (ValueError, TypeError) as error: raise RulesError(str(error)) from error return NativeRulesEvaluation( - findings=tuple(finding_from_row(row) for row in faults), + findings=tuple(faults), selected_codes=tuple(selected_codes), evaluated_models=evaluated, cache_hits=hits, @@ -128,49 +117,20 @@ def finalize_rules_rows( evaluated_codes: tuple[str, ...], findings: tuple[Finding, ...], ) -> tuple[Finding, ...]: - """Apply the exception policy natively to finding rows.""" + """Apply the exception policy natively, keeping each original finding's fixable mark.""" try: - rows: list[FindingRow] = _native.finalize_rule_findings_rows( + finalized: list[Finding] = _native.finalize_rule_findings_rows( { "project_dir": str(project_dir.resolve()), "config_json": _encoded(asdict(config)), "evaluated_codes": list(evaluated_codes), - "findings": [finding_row(finding) for finding in findings], + "findings": list(findings), } ) except (ValueError, TypeError) as error: raise RulesError(str(error)) from error - return tuple(finding_from_row(row) for row in rows) - - -def finding_row(finding: Finding) -> FindingRow: - """One finding as the native boundary's row.""" - - return ( - finding.unevaluated, - finding.code, - finding.path.as_posix(), - finding.line, - finding.column, - finding.message, - finding.remediation, - ) - - -def finding_from_row(row: FindingRow) -> Finding: - """One native finding row as a finding.""" - - unevaluated, code, path, line, column, message, remediation = row - return Finding( - code=code, - path=Path(path), - line=line, - column=column, - message=message, - remediation=remediation, - unevaluated=unevaluated, - ) + return tuple(finalized) def _encoded(value: object) -> bytes: @@ -178,67 +138,3 @@ def _encoded(value: object) -> bytes: return orjson.dumps(value, option=orjson.OPT_SORT_KEYS, default=str) except orjson.JSONEncodeError as error: raise RulesError(str(error)) from error - - -def _sql_test_row(test: CompiledSqlTest) -> tuple[object, ...]: - payload: object = ( - ( - _cte_rows(test.payload.authored_ctes), - _cte_rows(test.payload.expected_ctes), - _cte_rows(test.payload.assertion_ctes), - bool(test.payload.macro_mocks), - bool(test.payload.model_query_overrides), - ) - if isinstance(test.payload, CompiledModelSqlTestPayload) - else None - ) - case: object = ( - None - if test.case_name is None - else ( - test.parent_name, - test.case_name, - test.case_index, - test.case_fingerprint, - [ - (parameter.name, parameter.value_type.value, parameter.nullable) - for parameter in test.parameter_schema - ], - [(name, plain_typed_value(value)) for name, value in test.parameter_values], - ) - ) - return ( - (test.source_path or test.test_file.relative_path).as_posix(), - (test.ownership_root or test.test_file.ownership_root).as_posix(), - test.block_index or test.test_block.test_index, - test.name, - test.explicit_name, - test.mode.value, - ( - list(test.expected_model_names), - list(test.assertion_names), - list(test.assertion_target_model_names), - list(test.target_model_names), - ), - [(resource.kind.value, resource.name) for resource in test.tested_resources], - payload, - case, - ) - - -def _cte_rows(ctes: tuple[CompileSqlTestCte, ...]) -> list[tuple[str, str]]: - return [(cte.name, cte.sql_body) for cte in ctes] - - -def _scenario_row(scenario: CompiledSqlScenario) -> tuple[object, ...]: - description: object | None = scenario.scenario_file.header_values.get("description") - return ( - (scenario.source_path or scenario.scenario_file.relative_path).as_posix(), - (scenario.ownership_root or scenario.scenario_file.ownership_root).as_posix(), - scenario.name, - description if isinstance(description, str) else None, - list(scenario.expected_model_names), - list(scenario.assertion_names), - list(scenario.assertion_target_model_names), - list(scenario.target_model_names), - ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 914376ff72..a8530a8890 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -57,7 +57,6 @@ from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules from sqlbuild.rule_engine._helpers.engine.native import ( custom_rule_payloads, - decode_rule_finding, evaluate_native, finalize_native_findings, native_catalogue, @@ -742,15 +741,8 @@ def _cached_sql_findings( ) -> dict[str, tuple[Finding, ...]]: """Decoded stored findings of each path whose key still matches; undecodable ones miss.""" - hits: dict[str, str] - hits, _ = lookup_sql_rule_cache(entries=bucket, keys=list(keys.items())) - decoded: dict[str, tuple[Finding, ...]] = {} - for path, payload in hits.items(): - try: - decoded[path] = tuple(decode_rule_finding(value) for value in json.loads(payload)) - except (KeyError, RulesError, TypeError, ValueError): - continue - return decoded + hits: dict[str, list[Finding]] = lookup_sql_rule_cache(entries=bucket, keys=list(keys.items())) + return {path: tuple(findings) for path, findings in hits.items()} def _selected_model_paths( diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 23af930558..76752cffd9 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -440,9 +440,9 @@ class NativeRulesEvaluation: @dataclass(frozen=True) class CustomRulesOutcome: - """Custom-rule finding payloads in canonical order and their cache accounting.""" + """Custom-rule findings in canonical order and their cache accounting.""" - findings: tuple[dict[str, object], ...] + findings: tuple[Finding, ...] cache_hits: int cache_misses: int custom_ms: int diff --git a/src/sqlbuild/rule_engine/types.py b/src/sqlbuild/rule_engine/types.py index baf5012398..b74529ea48 100644 --- a/src/sqlbuild/rule_engine/types.py +++ b/src/sqlbuild/rule_engine/types.py @@ -26,7 +26,6 @@ type RuleOptionValue = bool | int | str | tuple[str, ...] | tuple[int, ...] type FactKey = tuple[str, ...] -type FindingRow = tuple[bool, str, str, int, int, str, str] type CustomEvaluationRow = tuple[ str, str, tuple[tuple[FactKey, ...], str | None] | None, list[dict[str, object]] ] diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 20fb1f390c..97d4b1e315 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -1,6 +1,7 @@ """Test helpers for native rules engine boundaries.""" import json +from collections.abc import Callable from dataclasses import dataclass, field from operator import attrgetter from pathlib import Path @@ -9,6 +10,7 @@ import pytest from sqlbuild._native import ( + NativeRulesRequest, project_with, ) from sqlbuild.compiler.compile.models import ( @@ -45,6 +47,9 @@ _MODEL_SQL: str = "WITH final AS (SELECT 1 AS id) SELECT id FROM final" +_BUILD_RULES_REQUEST: Callable[..., NativeRulesRequest] = native_rows._native.build_rules_request + + def project_with_scope(*, index: ScopeIndex) -> CompiledProjectView: """Return a minimal compiled project carrying canonical scope facts.""" @@ -66,13 +71,16 @@ def captured_native_request( def build_rules_request( project_rows: dict[str, Any], - models: list[tuple[object, ...]], - sql_tests: list[tuple[object, ...]], - sql_scenarios: list[tuple[object, ...]], - ) -> tuple[object, tuple[int, int, int]]: + models: tuple[object, list[str]], + sql_tests: list[object], + sql_scenarios: list[object], + ) -> object: + request: NativeRulesRequest = _BUILD_RULES_REQUEST( + project_rows, models, sql_tests, sql_scenarios + ) captured.update(cast(dict[str, Any], json.loads(project_rows["header_json"]))) - captured.update(models=models, sql_tests=sql_tests, sql_scenarios=sql_scenarios) - return object(), (len(models), len(sql_tests), len(sql_scenarios)) + captured.update(models=models, **json.loads(request.sql_facts_json())) + return request def evaluate_rules_request(request: object) -> NativeRulesEvaluation: return NativeRulesEvaluation( @@ -398,3 +406,46 @@ def _incremental_project(models: tuple[IncrementalModelSpec, ...]) -> CompiledPr return project_with( project=projects[0], models=tuple(project.models[0] for project in projects) ) + + +def sql_test_summary(fact: dict[str, Any]) -> tuple[object, ...]: + """The SQL test fact fields the native rules request carries, in a compact comparable form.""" + + return ( + fact["source_path"], + fact["ownership_root"], + fact["block_index"], + fact["name"], + fact["explicit_name"], + fact["mode"], + ( + fact["expected_model_names"], + fact["assertion_names"], + fact["assertion_target_model_names"], + fact["target_model_names"], + ), + [(resource["kind"], resource["name"]) for resource in fact["tested_resources"]], + ( + fact["authored_ctes"], + fact["expected_ctes"], + fact["assertion_ctes"], + fact["has_macro_mocks"], + fact["has_model_query_overrides"], + ), + fact["case_name"], + ) + + +def scenario_summary(fact: dict[str, Any]) -> tuple[object, ...]: + """The SQL scenario fact fields the native rules request carries, in field order.""" + + return ( + fact["source_path"], + fact["ownership_root"], + fact["name"], + fact["description"], + fact["expected_model_names"], + fact["assertion_names"], + fact["assertion_target_model_names"], + fact["target_model_names"], + ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py index 6906304ca6..08dae222f7 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py @@ -218,7 +218,7 @@ def test_given_untracked_rule_reading_hidden_file_when_file_changes_then_rule_re test_case.expected_rerun_hits, 2 - test_case.expected_rerun_hits, ) - assert tuple(finding["path"] for finding in after_edit.findings) == ( + assert tuple(finding.path.as_posix() for finding in after_edit.findings) == ( test_case.expected_finding_paths ) @@ -278,7 +278,7 @@ def run_host_then_edit(**kwargs: Any) -> object: 2 - test_case.expected_rerun_hits, ) assert rerun.findings == oracle.findings - assert tuple(finding["path"] for finding in rerun.findings) == ( + assert tuple(finding.path.as_posix() for finding in rerun.findings) == ( test_case.expected_finding_paths ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index 51c428b117..fab374960c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -22,6 +22,8 @@ captured_native_request, direct_sql_test, model_sql_test, + scenario_summary, + sql_test_summary, ) from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project @@ -86,7 +88,7 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar ) assert len(request["sql_tests"]) == test_case.expected_test_count - assert request["sql_tests"][:3] == [ + assert [sql_test_summary(fact) for fact in request["sql_tests"][:3]] == [ ( f"tests/unit/{name}.sql", "tests/unit", @@ -96,14 +98,14 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar mode, ([], [], [], []), [(mode, name)], - None, + ([], [], [], False, False), None, ) for index, (mode, name) in enumerate( (("macro", "normalize"), ("udf", "tax"), ("table_fn", "items")), start=1 ) ] - assert request["sql_tests"][3] == ( + assert sql_test_summary(request["sql_tests"][3]) == ( "tests/unit/test_orders__keeps_paid.sql", "tests/unit", 1, @@ -116,7 +118,7 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar None, ) assert len(request["sql_scenarios"]) == test_case.expected_scenario_count - assert request["sql_scenarios"] == [ + assert [scenario_summary(fact) for fact in request["sql_scenarios"]] == [ ( "tests/scenarios/orders.sql", "tests/scenarios", From d79c4b5adbbe66ae4e210b8e6aa3b15c01653f51 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:15:26 +0100 Subject: [PATCH 30/37] refactor(rules): separate rule-run thread and custom-rule dispatch modules Thread pools, futures and stop events around a rules run move to run/rule_threads.py, and custom-rule submission beside native evaluation moves to engine/custom_dispatch.py. Run planning, SQL rule evaluation and result finalisation stay in run/rules.py as plain functions. The two new modules and engine/custom_rules.py, which dispatches to the rule-host worker pool, are listed as user-code stays. --- python-boundary.toml | 15 ++ .../_helpers/engine/custom_dispatch.py | 180 ++++++++++++++ .../rule_engine/_helpers/engine/native.py | 166 +------------ .../rule_engine/_helpers/engine/ruleset.py | 2 +- .../rule_engine/_helpers/run/rule_threads.py | 152 ++++++++++++ .../rule_engine/_helpers/run/rules.py | 234 +++++++----------- .../rule_engine/main/_prepare_sql_rules.py | 2 +- src/sqlbuild/rule_engine/main/run_rules.py | 2 +- src/sqlbuild/rule_engine/models.py | 26 ++ .../sqlbuild/cli/commands/main/test_rules.py | 7 +- .../rule_engine/_helpers/engine/helpers.py | 2 +- .../rule_engine/_helpers/run/helpers.py | 2 +- .../_helpers/run/test_native_rows.py | 5 +- 13 files changed, 483 insertions(+), 312 deletions(-) create mode 100644 src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py create mode 100644 src/sqlbuild/rule_engine/_helpers/run/rule_threads.py diff --git a/python-boundary.toml b/python-boundary.toml index 6a00506e2a..40592263c0 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -104,6 +104,21 @@ path = "rule_engine/_helpers/host" reason = "user-code" note = "Subprocess host that loads and runs user rule modules." +[[stays]] +path = "rule_engine/_helpers/engine/custom_dispatch.py" +reason = "user-code" +note = "Submits custom rules to their worker thread beside native built-in rule evaluation." + +[[stays]] +path = "rule_engine/_helpers/engine/custom_rules.py" +reason = "user-code" +note = "Dispatches custom rules to the rule-host worker pool and digests the fact views they read." + +[[stays]] +path = "rule_engine/_helpers/run/rule_threads.py" +reason = "user-code" +note = "Runs rule phases and early SQL lint on worker threads with futures and stop events." + [[stays]] path = "rule_engine/classes/rule_context.py" reason = "user-code" diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py new file mode 100644 index 0000000000..4cdbe8af63 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py @@ -0,0 +1,180 @@ +"""Dispatch custom rules to their worker thread beside native built-in rule evaluation.""" + +from __future__ import annotations + +from collections.abc import Callable +from concurrent.futures import Executor, Future, ThreadPoolExecutor +from functools import partial +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity +from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached +from sqlbuild.rule_engine._helpers.engine.native import ( + custom_rule_payloads, + finalize_native_findings, + rule_selected, + selected_codes, +) +from sqlbuild.rule_engine._helpers.run.native_rows import ( + build_rules_request, + evaluate_rules_request, +) +from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES +from sqlbuild.rule_engine.models import ( + CustomRulesOutcome, + Finding, + NativeRulesEvaluation, + Rule, + RulesConfig, + RulesResult, +) + + +def evaluate_native( + *, + project: CompiledProjectView, + config: RulesConfig, + project_dir: Path, + catalogue: tuple[Rule, ...], + dialect: str = "generic", + initial_findings: tuple[Finding, ...] = (), + defer_suppressions: bool = False, + verify_determinism: bool = False, + custom_payloads: list[dict[str, object]] | None = None, + custom_outcome: Future[CustomRulesOutcome] | None = None, +) -> RulesResult: + """Evaluate built-in rules natively while custom rules run incrementally beside them.""" + + payloads: list[dict[str, object]] = ( + custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) + if custom_payloads is None + else custom_payloads + ) + return _evaluate_native_rows( + project=project, + config=config, + project_dir=project_dir, + dialect=dialect, + initial_findings=initial_findings, + defer_suppressions=defer_suppressions, + custom_payloads=payloads, + start_custom=( + (lambda **_: custom_outcome) + if custom_payloads is not None + else partial( + start_custom_rules, + project=project, + config=config, + project_dir=project_dir, + catalogue=catalogue, + custom_payloads=payloads, + dialect=dialect, + verify_determinism=verify_determinism, + ) + ), + ) + + +def _evaluate_native_rows( + *, + project: CompiledProjectView, + config: RulesConfig, + project_dir: Path, + dialect: str, + initial_findings: tuple[Finding, ...], + defer_suppressions: bool, + custom_payloads: list[dict[str, object]], + start_custom: Callable[..., Future[CustomRulesOutcome] | None], +) -> RulesResult: + request: _native.NativeRulesRequest = build_rules_request( + project=project, + config=config, + project_dir=project_dir, + dialect=dialect, + initial_findings=initial_findings, + custom_payloads=custom_payloads, + include_type_proof=any( + rule_selected(config=config, code=code) for code in TYPE_PROOF_RULE_CODES + ), + ) + with ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as pool: + custom_future: Future[CustomRulesOutcome] | None = start_custom(executor=pool) + native: NativeRulesEvaluation = evaluate_rules_request(request) + del request + custom: CustomRulesOutcome = ( + CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) + if custom_future is None + else custom_future.result() + ) + findings: tuple[Finding, ...] = ( + *native.findings, + *custom.findings, + ) + if not defer_suppressions: + findings = finalize_native_findings( + config=config, + project_dir=project_dir, + evaluated_codes=native.selected_codes, + findings=findings, + ) + return RulesResult( + findings=findings, + evaluated_models=native.evaluated_models, + cache_hits=(native.cache_hits + native.cache_misses if native.reused else native.cache_hits) + + custom.cache_hits, + cache_misses=(0 if native.reused else native.cache_misses) + custom.cache_misses, + built_in_ms=0 if native.reused else native.built_in_ms, + custom_ms=custom.custom_ms, + custom_cpu_ms=custom.custom_cpu_ms, + ) + + +def start_custom_rules( + *, + executor: Executor, + project: CompiledProjectView, + config: RulesConfig, + project_dir: Path, + catalogue: tuple[Rule, ...], + custom_payloads: list[dict[str, object]], + dialect: str, + verify_determinism: bool = False, +) -> Future[CustomRulesOutcome] | None: + """Submit selected custom rules after digesting installed code here, clear of GIL contention.""" + + custom_rules: tuple[Rule, ...] = _selected_custom_rules( + config=config, catalogue=catalogue, custom_payloads=custom_payloads + ) + if not custom_rules: + return None + if config.cache.enabled: + _ = compiled_code_identity() + return executor.submit( + evaluate_custom_rules_cached, + project=project, + config=config, + project_dir=project_dir.resolve(), + rules=custom_rules, + dialect=dialect, + verify_determinism=verify_determinism, + implementation_fingerprints={ + str(payload["code"]): str(payload["implementation_fingerprint"]) + for payload in custom_payloads + }, + ) + + +def _selected_custom_rules( + *, + config: RulesConfig, + catalogue: tuple[Rule, ...], + custom_payloads: list[dict[str, object]], +) -> tuple[Rule, ...]: + if not custom_payloads: + return () + selected: frozenset[str] = frozenset( + selected_codes(config=config, custom_payloads=custom_payloads) + ) + return tuple(rule for rule in catalogue if rule.custom and rule.code in selected) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 7c5d5c1544..604eaa9c9d 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -4,10 +4,7 @@ import inspect import json -from collections.abc import Callable -from concurrent.futures import Executor, Future, ThreadPoolExecutor from dataclasses import asdict -from functools import partial from pathlib import Path import sqlbuild._native as _native @@ -16,177 +13,20 @@ custom_rule_import_closure, custom_rule_test_evidence, ) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.run.native_rows import ( - build_rules_request, - evaluate_rules_request, finalize_rules_rows, ) from sqlbuild.rule_engine.constants import ( RULES_NATIVE_API_VERSION, - TYPE_PROOF_RULE_CODES, ) from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import ( - CustomRulesOutcome, Finding, - NativeRulesEvaluation, Rule, RulesConfig, - RulesResult, ) -def evaluate_native( - *, - project: CompiledProjectView, - config: RulesConfig, - project_dir: Path, - catalogue: tuple[Rule, ...], - dialect: str = "generic", - initial_findings: tuple[Finding, ...] = (), - defer_suppressions: bool = False, - verify_determinism: bool = False, - custom_payloads: list[dict[str, object]] | None = None, - custom_outcome: Future[CustomRulesOutcome] | None = None, -) -> RulesResult: - """Evaluate built-in rules natively while custom rules run incrementally beside them.""" - - payloads: list[dict[str, object]] = ( - custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) - if custom_payloads is None - else custom_payloads - ) - return _evaluate_native_rows( - project=project, - config=config, - project_dir=project_dir, - dialect=dialect, - initial_findings=initial_findings, - defer_suppressions=defer_suppressions, - custom_payloads=payloads, - start_custom=( - (lambda **_: custom_outcome) - if custom_payloads is not None - else partial( - start_custom_rules, - project=project, - config=config, - project_dir=project_dir, - catalogue=catalogue, - custom_payloads=payloads, - dialect=dialect, - verify_determinism=verify_determinism, - ) - ), - ) - - -def _evaluate_native_rows( - *, - project: CompiledProjectView, - config: RulesConfig, - project_dir: Path, - dialect: str, - initial_findings: tuple[Finding, ...], - defer_suppressions: bool, - custom_payloads: list[dict[str, object]], - start_custom: Callable[..., Future[CustomRulesOutcome] | None], -) -> RulesResult: - request: _native.NativeRulesRequest = build_rules_request( - project=project, - config=config, - project_dir=project_dir, - dialect=dialect, - initial_findings=initial_findings, - custom_payloads=custom_payloads, - include_type_proof=any( - _rule_selected(config=config, code=code) for code in TYPE_PROOF_RULE_CODES - ), - ) - with ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as pool: - custom_future: Future[CustomRulesOutcome] | None = start_custom(executor=pool) - native: NativeRulesEvaluation = evaluate_rules_request(request) - del request - custom: CustomRulesOutcome = ( - CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) - if custom_future is None - else custom_future.result() - ) - findings: tuple[Finding, ...] = ( - *native.findings, - *custom.findings, - ) - if not defer_suppressions: - findings = finalize_native_findings( - config=config, - project_dir=project_dir, - evaluated_codes=native.selected_codes, - findings=findings, - ) - return RulesResult( - findings=findings, - evaluated_models=native.evaluated_models, - cache_hits=(native.cache_hits + native.cache_misses if native.reused else native.cache_hits) - + custom.cache_hits, - cache_misses=(0 if native.reused else native.cache_misses) + custom.cache_misses, - built_in_ms=0 if native.reused else native.built_in_ms, - custom_ms=custom.custom_ms, - custom_cpu_ms=custom.custom_cpu_ms, - ) - - -def start_custom_rules( - *, - executor: Executor, - project: CompiledProjectView, - config: RulesConfig, - project_dir: Path, - catalogue: tuple[Rule, ...], - custom_payloads: list[dict[str, object]], - dialect: str, - verify_determinism: bool = False, -) -> Future[CustomRulesOutcome] | None: - """Submit selected custom rules after digesting installed code here, clear of GIL contention.""" - - custom_rules: tuple[Rule, ...] = _selected_custom_rules( - config=config, catalogue=catalogue, custom_payloads=custom_payloads - ) - if not custom_rules: - return None - if config.cache.enabled: - _ = compiled_code_identity() - return executor.submit( - evaluate_custom_rules_cached, - project=project, - config=config, - project_dir=project_dir.resolve(), - rules=custom_rules, - dialect=dialect, - verify_determinism=verify_determinism, - implementation_fingerprints={ - str(payload["code"]): str(payload["implementation_fingerprint"]) - for payload in custom_payloads - }, - ) - - -def _selected_custom_rules( - *, - config: RulesConfig, - catalogue: tuple[Rule, ...], - custom_payloads: list[dict[str, object]], -) -> tuple[Rule, ...]: - if not custom_payloads: - return () - selected: frozenset[str] = frozenset( - _selected_codes(config=config, custom_payloads=custom_payloads) - ) - return tuple(rule for rule in catalogue if rule.custom and rule.code in selected) - - def finalize_native_findings( *, config: RulesConfig, @@ -231,7 +71,7 @@ def native_selected_codes( ) -> tuple[str, ...]: """Resolve the active ruleset through the native Fensu adapter.""" - return _selected_codes( + return selected_codes( config=config, custom_payloads=( custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) @@ -241,7 +81,7 @@ def native_selected_codes( ) -def _selected_codes( +def selected_codes( *, config: RulesConfig, custom_payloads: list[dict[str, object]] ) -> tuple[str, ...]: request: dict[str, object] = { @@ -279,7 +119,7 @@ def _config_payload(config: RulesConfig) -> dict[str, object]: return asdict(config) -def _rule_selected(*, config: RulesConfig, code: str) -> bool: +def rule_selected(*, config: RulesConfig, code: str) -> bool: selected: bool = any(code.startswith(selector) for selector in config.select) ignored: bool = any(code.startswith(selector) for selector in config.ignore) return selected and not ignored diff --git a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py b/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py index 5ff0195d61..1b1625c074 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py @@ -13,8 +13,8 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors +from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine._helpers.engine.native import evaluate_native from sqlbuild.rule_engine.models import ResolvedRuleset, Rule, RulesConfig, RulesResult diff --git a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py new file mode 100644 index 0000000000..8ae88128f3 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py @@ -0,0 +1,152 @@ +"""Thread and future handling around one rules run and early SQL lint.""" + +from __future__ import annotations + +from concurrent.futures import Executor, Future, ThreadPoolExecutor +from contextvars import Token +from pathlib import Path +from threading import Event + +from sqlbuild.compiler.compile.models import ( + CompiledObjectKey, + CompiledSqlExpansion, + DeclarationScopeBuild, + SqlExpansionContext, +) +from sqlbuild.compiler.compile.types import CompileInputsView +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.lint.classes.stop_context import LintStopContext +from sqlbuild.lint.main.build_expansion_context import build_expansion_context +from sqlbuild.lint.main.run_lint import run_lint +from sqlbuild.lint.models import LintConfig, LintRunResult +from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native, start_custom_rules +from sqlbuild.rule_engine._helpers.run.rules import ( + early_lint_expansions, + finish_rule_run, + plan_rule_run, + run_planned_sql_rules, + sql_lint_preparation, +) +from sqlbuild.rule_engine.models import ( + CustomRulesOutcome, + PreparedSqlLint, + PreparedSqlLintResult, + RuleRunPlan, + RulesConfig, + RulesResult, + RulesRunResult, + SqlExpansionReuse, + SqlRulesEvaluation, +) + + +def evaluate_rules( + *, + graph: ProjectGraph, + discovered_inputs: DiscoveredProjectInputs, + config: RulesConfig, + project_dir: Path, + dialect: str, + selected_keys: frozenset[CompiledObjectKey] | None = None, + prepared_sql: PreparedSqlLint | None = None, + expansion_reuse: SqlExpansionReuse | None = None, + no_sql_analysis: bool = False, +) -> RulesRunResult: + """Evaluate independent rule phases concurrently, then finalize their combined findings.""" + + plan: RuleRunPlan = plan_rule_run( + graph=graph, + config=config, + project_dir=project_dir, + selected_keys=selected_keys, + no_sql_analysis=no_sql_analysis, + ) + with ( + ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as custom, + ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-rules") as executor, + ): + custom_outcome: Future[CustomRulesOutcome] | None = start_custom_rules( + executor=custom, + project=plan.project, + config=plan.config, + project_dir=plan.project_dir, + catalogue=plan.catalogue, + custom_payloads=plan.custom_payloads, + dialect=dialect, + ) + native_result: Future[RulesResult] = executor.submit( + evaluate_native, + project=plan.project, + config=plan.config, + project_dir=plan.project_dir, + catalogue=plan.catalogue, + dialect=dialect, + defer_suppressions=True, + custom_payloads=plan.custom_payloads, + custom_outcome=custom_outcome, + ) + sql_result: SqlRulesEvaluation = run_planned_sql_rules( + plan=plan, + graph=graph, + discovered_inputs=discovered_inputs, + dialect=dialect, + prepared_sql=prepared_sql, + expansion_reuse=expansion_reuse, + ) + result: RulesResult = native_result.result() + return finish_rule_run(plan=plan, sql_result=sql_result, result=result) + + +def prepare_sql_rules( + *, inputs: CompileInputsView, executor: Executor, dialect: str, stop: Event +) -> PreparedSqlLint | None: + """Start early SQL lint on `executor` when the project selects SQL rules without a cache.""" + + preparation: tuple[Path, tuple[str, ...], LintConfig] | None = sql_lint_preparation( + inputs=inputs, dialect=dialect + ) + if preparation is None: + return None + project_dir, codes, config = preparation + return PreparedSqlLint( + codes=codes, + project_dir=project_dir, + future=executor.submit( + _prepare_sql_lint, + project_dir=project_dir, + config=config, + discovered_inputs=inputs.discovered_inputs, + static_declaration_scope=inputs.declaration_scope, + compiled_expansions=early_lint_expansions(inputs.model_inputs), + stop=stop, + ), + ) + + +def _prepare_sql_lint( + *, + project_dir: Path, + config: LintConfig, + discovered_inputs: DiscoveredProjectInputs, + static_declaration_scope: DeclarationScopeBuild | None, + compiled_expansions: dict[Path, CompiledSqlExpansion], + stop: Event, +) -> PreparedSqlLintResult: + stop_token: Token[Event | None] = LintStopContext.active.set(stop) + try: + context: SqlExpansionContext = build_expansion_context( + project_dir=project_dir, + discovered_inputs=discovered_inputs, + static_declaration_scope=static_declaration_scope, + ) + result: LintRunResult = run_lint( + project_dir=project_dir, + config=config, + discovered_inputs=discovered_inputs, + compiled_expansions=compiled_expansions, + expansion_context=context, + ) + finally: + LintStopContext.active.reset(stop_token) + return PreparedSqlLintResult(result=result, context=context) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index a8530a8890..9f5edda053 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -4,11 +4,8 @@ import json import time -from concurrent.futures import Executor, Future, ThreadPoolExecutor -from contextvars import Token from dataclasses import asdict, dataclass, replace from pathlib import Path -from threading import Event from sqlbuild._native import ( encode_sql_rule_findings, @@ -36,7 +33,6 @@ ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.constants import ( DEFAULT_MAX_LITERAL_LENGTH, DEFAULT_MAX_RANKING_ORDER_BY, @@ -57,10 +53,8 @@ from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules from sqlbuild.rule_engine._helpers.engine.native import ( custom_rule_payloads, - evaluate_native, finalize_native_findings, native_catalogue, - start_custom_rules, ) from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings @@ -69,15 +63,16 @@ from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import ( - CustomRulesOutcome, Finding, PreparedSqlLint, PreparedSqlLintResult, Rule, + RuleRunPlan, RulesConfig, RulesResult, RulesRunResult, SqlExpansionReuse, + SqlRulesEvaluation, ) _SQL_RULE_SUPPRESSION_CODE: str = "SQBRSQL000" @@ -89,31 +84,20 @@ class _SqlRuleInputs: project_files: dict[Path, str] | None -@dataclass(frozen=True) -class _SqlRulesEvaluation: - findings: tuple[Finding, ...] - cache_hits: int - cache_misses: int - - -def evaluate_rules( +def plan_rule_run( *, graph: ProjectGraph, - discovered_inputs: DiscoveredProjectInputs, config: RulesConfig, project_dir: Path, - dialect: str, - selected_keys: frozenset[CompiledObjectKey] | None = None, - prepared_sql: PreparedSqlLint | None = None, - expansion_reuse: SqlExpansionReuse | None = None, - no_sql_analysis: bool = False, -) -> RulesRunResult: - """Evaluate independent rule phases concurrently, then finalize their combined findings.""" + selected_keys: frozenset[CompiledObjectKey] | None, + no_sql_analysis: bool, +) -> RuleRunPlan: + """Build the catalogue and select the rules, project slice and settings one run evaluates.""" + release_freed_memory() effective_config: RulesConfig = resolve_rule_ignore_selectors( config=config, project=graph.project ) - resolved_project_dir: Path = project_dir.resolve() include_custom: bool = _config_references_custom_rules(effective_config) catalogue: tuple[Rule, ...] = build_catalogue( @@ -136,7 +120,6 @@ def evaluate_rules( rule.code for rule in selected if rule.code in TYPE_PROOF_RULE_CODES ) selected = tuple(rule for rule in selected if rule.code not in TYPE_PROOF_RULE_CODES) - native_rules: tuple[Rule, ...] = tuple(rule for rule in selected if not rule.custom) custom_rules: tuple[Rule, ...] = tuple(rule for rule in selected if rule.custom) if custom_rules: _ = verify_custom_rules(rules=custom_rules, project_dir=resolved_project_dir) @@ -153,64 +136,71 @@ def evaluate_rules( select=tuple(rule.code for rule in selected), ignore=(), ) - with ( - ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as custom, - ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-rules") as executor, - ): - custom_outcome: Future[CustomRulesOutcome] | None = start_custom_rules( - executor=custom, - project=selected_project, - config=selected_config, - project_dir=resolved_project_dir, - catalogue=catalogue, - custom_payloads=custom_payloads, - dialect=dialect, - ) - native_result: Future[RulesResult] = executor.submit( - evaluate_native, - project=selected_project, - config=selected_config, - project_dir=resolved_project_dir, - catalogue=catalogue, - dialect=dialect, - defer_suppressions=True, - custom_payloads=custom_payloads, - custom_outcome=custom_outcome, - ) - lint_config: LintConfig = sql_lint_config( - dialect=dialect, - codes=_sql_rule_codes(native_rules), - thresholds=effective_config.thresholds, - relation_keys=compiled_relation_keys(graph.project), - allow_suppressions=effective_config.allow_exceptions, - ) - sql_rule_inputs: _SqlRuleInputs = _prepare_sql_rule_inputs( - rules=native_rules, - project=selected_project, - dialect=sql_rule_lint_key(config=lint_config), - project_dir=resolved_project_dir, - collect_files=prepared_sql is None and model_paths is None, - discovered_inputs=discovered_inputs, - ) - sql_started: float = time.monotonic() - sql_result: _SqlRulesEvaluation = _run_sql_rules( - rules=native_rules, - project_dir=resolved_project_dir, - discovered_inputs=discovered_inputs, - project=selected_project, - lint_config=lint_config, - selected_model_paths=model_paths, - cache_enabled=effective_config.cache.enabled, - prepared_sql=prepared_sql, - expansion_reuse=expansion_reuse, - prepared_inputs=sql_rule_inputs, - ) - sql_ms: int = round((time.monotonic() - sql_started) * 1000) - result: RulesResult = native_result.result() - finalized: tuple[Finding, ...] = finalize_native_findings( - config=selected_config, + return RuleRunPlan( + effective_config=effective_config, project_dir=resolved_project_dir, - evaluated_codes=tuple(rule.code for rule in selected), + catalogue=catalogue, + custom_payloads=custom_payloads, + selected=selected, + skipped_type_proof_rules=skipped_type_proof_rules, + model_paths=model_paths, + project=selected_project, + config=selected_config, + ) + + +def run_planned_sql_rules( + *, + plan: RuleRunPlan, + graph: ProjectGraph, + discovered_inputs: DiscoveredProjectInputs, + dialect: str, + prepared_sql: PreparedSqlLint | None, + expansion_reuse: SqlExpansionReuse | None, +) -> SqlRulesEvaluation: + """Run the planned SQL rules on the caller while built-in rules evaluate natively.""" + + native_rules: tuple[Rule, ...] = tuple(rule for rule in plan.selected if not rule.custom) + lint_config: LintConfig = sql_lint_config( + dialect=dialect, + codes=_sql_rule_codes(native_rules), + thresholds=plan.effective_config.thresholds, + relation_keys=compiled_relation_keys(graph.project), + allow_suppressions=plan.effective_config.allow_exceptions, + ) + sql_rule_inputs: _SqlRuleInputs = _prepare_sql_rule_inputs( + rules=native_rules, + project=plan.project, + dialect=sql_rule_lint_key(config=lint_config), + project_dir=plan.project_dir, + collect_files=prepared_sql is None and plan.model_paths is None, + discovered_inputs=discovered_inputs, + ) + sql_started: float = time.monotonic() + sql_result: SqlRulesEvaluation = _run_sql_rules( + rules=native_rules, + project_dir=plan.project_dir, + discovered_inputs=discovered_inputs, + project=plan.project, + lint_config=lint_config, + selected_model_paths=plan.model_paths, + cache_enabled=plan.effective_config.cache.enabled, + prepared_sql=prepared_sql, + expansion_reuse=expansion_reuse, + prepared_inputs=sql_rule_inputs, + ) + return replace(sql_result, sql_ms=round((time.monotonic() - sql_started) * 1000)) + + +def finish_rule_run( + *, plan: RuleRunPlan, sql_result: SqlRulesEvaluation, result: RulesResult +) -> RulesRunResult: + """Finalize the combined SQL, built-in and custom findings of one run.""" + + finalized: tuple[Finding, ...] = finalize_native_findings( + config=plan.config, + project_dir=plan.project_dir, + evaluated_codes=tuple(rule.code for rule in plan.selected), findings=(*sql_result.findings, *result.findings), ) findings: tuple[Finding, ...] = tuple( @@ -226,16 +216,16 @@ def evaluate_rules( result.evaluated_models - len( {item.path for item in findings if item.unevaluated} - & {model.relative_path for model in selected_project.models} + & {model.relative_path for model in plan.project.models} ), ), unevaluated_resources=len({item.path for item in findings if item.unevaluated}), - built_in_ms=sql_ms + result.built_in_ms, + built_in_ms=sql_result.sql_ms + result.built_in_ms, custom_ms=result.custom_ms, custom_cpu_ms=result.custom_cpu_ms, cache_hits=result.cache_hits + sql_result.cache_hits, cache_misses=result.cache_misses + sql_result.cache_misses, - skipped_type_proof_rules=skipped_type_proof_rules, + skipped_type_proof_rules=plan.skipped_type_proof_rules, ) @@ -279,9 +269,11 @@ def _config_references_custom_rules(config: RulesConfig) -> bool: return any(selector.startswith("XSQBR") for selector in selectors) -def prepare_sql_rules( - *, inputs: CompileInputsView, executor: Executor, dialect: str, stop: Event -) -> PreparedSqlLint | None: +def sql_lint_preparation( + *, inputs: CompileInputsView, dialect: str +) -> tuple[Path, tuple[str, ...], LintConfig] | None: + """The project, SQL rule codes and lint settings early SQL lint runs with, if it should run.""" + project_dir: Path | None = inputs.discovered_inputs.project_dir if project_dir is None or any(diagnostic.is_error for diagnostic in inputs.diagnostics): return None @@ -294,27 +286,19 @@ def prepare_sql_rules( codes: tuple[str, ...] = _selected_sql_codes(config) if not codes: return None - return PreparedSqlLint( - codes=codes, - project_dir=project_dir, - future=executor.submit( - _prepare_sql_lint, - project_dir=project_dir, - config=sql_lint_config( - dialect=dialect, - codes=codes, - thresholds=config.thresholds, - allow_suppressions=config.allow_exceptions, - relation_keys=declared_relation_keys( - model_rows=(model.relation_key_row for model in inputs.model_inputs), - seeds=(seed.schema_entry for seed in inputs.seed_inputs), - sources=(source.source_entry for source in inputs.source_inputs), - ), + return ( + project_dir, + codes, + sql_lint_config( + dialect=dialect, + codes=codes, + thresholds=config.thresholds, + allow_suppressions=config.allow_exceptions, + relation_keys=declared_relation_keys( + model_rows=(model.relation_key_row for model in inputs.model_inputs), + seeds=(seed.schema_entry for seed in inputs.seed_inputs), + sources=(source.source_entry for source in inputs.source_inputs), ), - discovered_inputs=inputs.discovered_inputs, - static_declaration_scope=inputs.declaration_scope, - compiled_expansions=early_lint_expansions(inputs.model_inputs), - stop=stop, ), ) @@ -332,34 +316,6 @@ def early_lint_expansions( return expansions -def _prepare_sql_lint( - *, - project_dir: Path, - config: LintConfig, - discovered_inputs: DiscoveredProjectInputs, - static_declaration_scope: DeclarationScopeBuild | None, - compiled_expansions: dict[Path, CompiledSqlExpansion], - stop: Event, -) -> PreparedSqlLintResult: - stop_token: Token[Event | None] = LintStopContext.active.set(stop) - try: - context: SqlExpansionContext = build_expansion_context( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - static_declaration_scope=static_declaration_scope, - ) - result: LintRunResult = run_lint( - project_dir=project_dir, - config=config, - discovered_inputs=discovered_inputs, - compiled_expansions=compiled_expansions, - expansion_context=context, - ) - finally: - LintStopContext.active.reset(stop_token) - return PreparedSqlLintResult(result=result, context=context) - - def _selection_rules_config( *, config: RulesConfig, @@ -489,10 +445,10 @@ def _run_sql_rules( prepared_sql: PreparedSqlLint | None = None, expansion_reuse: SqlExpansionReuse | None = None, prepared_inputs: _SqlRuleInputs | None = None, -) -> _SqlRulesEvaluation: +) -> SqlRulesEvaluation: codes: tuple[str, ...] = _sql_rule_codes(rules) if not codes: - return _SqlRulesEvaluation(findings=(), cache_hits=0, cache_misses=0) + return SqlRulesEvaluation(findings=(), cache_hits=0, cache_misses=0) models_by_path: dict[str, CompiledModelView] = _sql_rule_models(project) bucket: dict[str, tuple[str, str]] = _read_sql_rule_cache(project_dir) if cache_enabled else {} identities: dict[str, str] = ( @@ -616,7 +572,7 @@ def _run_sql_rules( for path, identity in written_identities.items(): bucket[path] = (identity, encode_sql_rule_findings(findings=by_path[path])) write_sql_rule_cache(path=_sql_rule_cache_path(project_dir), entries=bucket) - return _SqlRulesEvaluation( + return SqlRulesEvaluation( findings=with_duplicate_literal_hints(findings=tuple(findings), project_dir=project_dir), cache_hits=len(identities) - len(misses), cache_misses=len(misses), diff --git a/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py b/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py index d2cb8c0d2e..c9be51978e 100644 --- a/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py +++ b/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py @@ -6,7 +6,7 @@ from threading import Event from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.rule_engine._helpers.run.rules import prepare_sql_rules as prepare +from sqlbuild.rule_engine._helpers.run.rule_threads import prepare_sql_rules as prepare from sqlbuild.rule_engine.models import PreparedSqlLint diff --git a/src/sqlbuild/rule_engine/main/run_rules.py b/src/sqlbuild/rule_engine/main/run_rules.py index c5ecda7344..ea50bdb95a 100644 --- a/src/sqlbuild/rule_engine/main/run_rules.py +++ b/src/sqlbuild/rule_engine/main/run_rules.py @@ -5,7 +5,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.rule_engine._helpers.run.rules import evaluate_rules +from sqlbuild.rule_engine._helpers.run.rule_threads import evaluate_rules from sqlbuild.rule_engine.models import ( PreparedSqlLint, RulesConfig, diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 76752cffd9..610da26686 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -9,6 +9,7 @@ from typing import Any from sqlbuild.compiler.compile.models import DeclarationScopeBuild, SqlExpansionContext +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint.models import LintRunResult from sqlbuild.rule_engine.types import FactKey, RuleCheck, RuleOptionValue, RuleSubject @@ -339,6 +340,31 @@ class PreparedSqlLint: project_dir: Path | None = None +@dataclass(frozen=True) +class SqlRulesEvaluation: + """SQL rule findings of one run, their cache accounting and elapsed time.""" + + findings: tuple[Finding, ...] + cache_hits: int + cache_misses: int + sql_ms: int = 0 + + +@dataclass(frozen=True) +class RuleRunPlan: + """The catalogue, selected rules, project slice and settings one rules run evaluates.""" + + effective_config: RulesConfig + project_dir: Path + catalogue: tuple[Rule, ...] + custom_payloads: list[dict[str, object]] + selected: tuple[Rule, ...] + skipped_type_proof_rules: tuple[str, ...] + model_paths: frozenset[str] | None + project: CompiledProjectView + config: RulesConfig + + @dataclass(frozen=True) class RulesRunResult: """Unified findings and phase measurements for configured rules.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 40506fd311..68aecdd422 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -15,6 +15,7 @@ import sqlbuild.compiler.compile._helpers.render.macros as macros_module import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module +import sqlbuild.rule_engine._helpers.run.rule_threads as rule_threads_module import sqlbuild.rule_engine._helpers.run.rules as rules_module from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro @@ -25,11 +26,11 @@ from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostSplitIntegrationTestCase, + CustomRuleCacheSequenceTestCase, DynamicPivotRulesIntegrationTestCase, EarlyLintInputsTestCase, ExplicitContractOutputRuleIntegrationTestCase, ImplicitAliasRuleIntegrationTestCase, - CustomRuleCacheSequenceTestCase, NumericRangeDecisionIntegrationTestCase, RulePassIntegrationTestCase, RulesIntegrationTestCase, @@ -1046,7 +1047,7 @@ def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_ ) lint_started: Event = Event() stop_observed: list[bool] = [] - original_run_lint: Callable[..., LintRunResult] = rules_module.run_lint + original_run_lint: Callable[..., LintRunResult] = rule_threads_module.run_lint def stop_awaiting_run_lint(**kwargs: Any) -> LintRunResult: lint_started.set() @@ -1054,7 +1055,7 @@ def stop_awaiting_run_lint(**kwargs: Any) -> LintRunResult: stop_observed.append(stop is not None and stop.wait(timeout=_EARLY_LINT_STOP_TIMEOUT_S)) return original_run_lint(**kwargs) - monkeypatch.setattr(rules_module, "run_lint", stop_awaiting_run_lint) + monkeypatch.setattr(rule_threads_module, "run_lint", stop_awaiting_run_lint) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 97d4b1e315..6a5bc8a547 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -24,7 +24,7 @@ from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.rule_engine._helpers.engine import native +from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.run import native_rows diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py index cb858b6cd6..141ef13ceb 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py @@ -3,7 +3,7 @@ import pytest from sqlbuild._native import NativeRulesRequest -from sqlbuild.rule_engine._helpers.engine import native +from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native from sqlbuild.rule_engine._helpers.run import native_rows from sqlbuild.rule_engine.models import NativeRulesEvaluation diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py index 51c7aff77b..ce522e41d0 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py @@ -5,7 +5,8 @@ import pytest from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine import native +from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native +from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import RulesCacheConfig, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule from tests.unit.src.sqlbuild.rule_engine._helpers.run._test_types import ( @@ -42,7 +43,7 @@ def test_given_config_value_the_encoder_rejects_when_building_natively_then_rule config_values={"priority": test_case.rejected_value}, ) - with pytest.raises(native.RulesError, match=test_case.expected_message): + with pytest.raises(RulesError, match=test_case.expected_message): native.evaluate_native( project=project, config=RulesConfig(cache=RulesCacheConfig(enabled=False)), From 435186741290f6647a4538d15fe7692329f3dca3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:24:54 +0100 Subject: [PATCH 31/37] refactor(rules): build the built-in rule catalogue natively Built-in Rule models, their declared options and guidance are built by a native builtin_rules binding from the sqlbuild-rules catalogue, and the early-lint SQL code selection runs natively. The JSON catalogue hand-off and the Python option table are removed. Loading custom rule modules and validating their declared options stay in Python with the user code. --- .../_helpers/rules/builtin_catalogue.rs | 143 ++++++++++++++++++ .../src/bindings/_helpers/rules/evaluation.rs | 16 +- .../src/bindings/_helpers/rules/mod.rs | 1 + .../bindings/_helpers/rules/registration.rs | 5 +- crates/sqlbuild-rules/src/constants.rs | 1 + crates/sqlbuild-rules/src/models.rs | 6 - .../src/rules/main/builtin_options.rs | 53 +++++++ crates/sqlbuild-rules/src/rules/main/mod.rs | 2 + .../src/rules/main/selected_sql_codes.rs | 29 ++++ crates/sqlbuild-rules/src/rules/models.rs | 17 +++ src/sqlbuild/_native.pyi | 4 +- .../rule_engine/_helpers/engine/catalogue.py | 112 +------------- .../rule_engine/_helpers/engine/native.py | 18 --- .../rule_engine/_helpers/run/rules.py | 13 +- src/sqlbuild/rule_engine/constants.py | 4 - 15 files changed, 258 insertions(+), 166 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs create mode 100644 crates/sqlbuild-rules/src/rules/main/builtin_options.rs create mode 100644 crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs new file mode 100644 index 0000000000..edbec57e0a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs @@ -0,0 +1,143 @@ +//! The built-in rule catalogue returned as the Python `Rule` models. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::models::RuleMetadata; +use sqlbuild_rules::rules::main::builtin_options::builtin_options; +use sqlbuild_rules::rules::main::catalogue::catalogue; +use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; +use sqlbuild_rules::rules::models::{BuiltinOption, BuiltinOptionKind}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const TYPES_MODULE: &str = "sqlbuild.rule_engine.types"; +const FAMILY_SUFFIX_LENGTH: usize = 3; + +fn keywords<'py>( + py: Python<'py>, + fields: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in fields { + arguments.set_item(name, value)?; + } + Ok(arguments) +} + +fn option<'py>( + option_class: &Bound<'py, PyAny>, + declared: BuiltinOption, +) -> PyResult> { + let py: Python<'py> = option_class.py(); + let (factory, default): (&str, Bound<'py, PyAny>) = match declared.kind { + BuiltinOptionKind::Boolean => ("boolean", PyBool::new(py, false).to_owned().into_any()), + BuiltinOptionKind::StringList => ("string_list", PyTuple::empty(py).into_any()), + }; + let arguments = keywords( + py, + vec![ + ("name", PyString::new(py, declared.name).into_any()), + ("default", default), + ( + "description", + PyString::new(py, declared.description).into_any(), + ), + ], + )?; + option_class.call_method(factory, (), Some(&arguments)) +} + +fn rule<'py>( + models: &Bound<'py, PyModule>, + subjects: &Bound<'py, PyAny>, + check: &Bound<'py, PyAny>, + metadata: RuleMetadata, +) -> PyResult> { + let py: Python<'py> = models.py(); + let option_class: Bound<'py, PyAny> = models.getattr("RuleOption")?; + let options: Vec> = builtin_options(&metadata.code) + .into_iter() + .map(|declared| option(&option_class, declared)) + .collect::>()?; + let guidance: Bound<'py, PyAny> = match metadata.guidance { + Some(guidance) => models.getattr("RuleGuidance")?.call( + (), + Some(&keywords( + py, + vec![ + ( + "good_example", + PyString::new(py, &guidance.good_example).into_any(), + ), + ( + "anti_tautology", + PyString::new(py, &guidance.anti_tautology).into_any(), + ), + ( + "mutation_check", + PyString::new(py, &guidance.mutation_check).into_any(), + ), + ], + )?), + )?, + None => py.None().into_bound(py), + }; + let family: &str = &metadata.code[..metadata.code.len().saturating_sub(FAMILY_SUFFIX_LENGTH)]; + let subject: Bound<'py, PyAny> = subjects.getattr(if metadata.project_wide { + "PROJECT" + } else { + "MODEL" + })?; + let text = |value: &str| PyString::new(py, value).into_any(); + let arguments = keywords( + py, + vec![ + ("code", text(&metadata.code)), + ("family", text(family)), + ("slug", text(&metadata.slug)), + ("message", text(&metadata.message)), + ("remediation", text(&metadata.remediation)), + ("check", check.clone()), + ("options", PyTuple::new(py, options)?.into_any()), + ( + "enabled_by_default", + PyBool::new(py, metadata.enabled_by_default) + .to_owned() + .into_any(), + ), + ("subject", subject), + ("guidance", guidance), + ], + )?; + models.getattr("Rule")?.call((), Some(&arguments)) +} + +/// Every built-in rule as a `Rule` whose `check` is the given placeholder. +#[pyfunction] +#[pyo3(signature = (*, check))] +fn builtin_rules<'py>( + py: Python<'py>, + check: Bound<'py, PyAny>, +) -> PyResult>> { + let models: Bound<'py, PyModule> = py.import(MODELS_MODULE)?; + let subjects: Bound<'py, PyAny> = py.import(TYPES_MODULE)?.getattr("RuleSubject")?; + compiler_guard(|| Ok(catalogue()))? + .into_iter() + .map(|metadata| rule(&models, &subjects, &check, metadata)) + .collect() +} + +/// The built-in SQL rule codes `select` enables and `ignore` does not, in catalogue order. +#[pyfunction] +#[pyo3(signature = (*, select, ignore))] +fn selected_sql_rule_codes(select: Vec, ignore: Vec) -> PyResult> { + compiler_guard(|| Ok(selected_sql_codes(&select, &ignore))) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(builtin_rules, module)?)?; + module.add_function(wrap_pyfunction!(selected_sql_rule_codes, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs index 188c8977dd..30ab858b7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs @@ -2,9 +2,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_rules::constants::API_VERSION; -use sqlbuild_rules::models::CatalogueResponse; -use sqlbuild_rules::rules::main::{catalogue, selected_codes}; +use sqlbuild_rules::rules::main::selected_codes; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::types::CompilerDetach; @@ -17,17 +15,6 @@ fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { .map_err(value_error) } -#[pyfunction] -fn catalogue_json() -> PyResult { - compiler_guard(|| { - serde_json::to_string(&CatalogueResponse { - version: API_VERSION, - rules: catalogue::catalogue(), - }) - .map_err(value_error) - }) -} - #[pyfunction] fn selected_codes_json(request_json: &str) -> PyResult { compiler_guard(|| selected_codes::selected_codes_json(request_json).map_err(value_error)) @@ -35,7 +22,6 @@ fn selected_codes_json(request_json: &str) -> PyResult { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; - module.add_function(wrap_pyfunction!(catalogue_json, module)?)?; module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?; crate::bindings::_helpers::rules::rows::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index e0119dfd93..57a21d4b9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod builtin_catalogue; pub(crate) mod custom_rule_cache; pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 0ac03ac4cc..f07045325b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -3,8 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ - custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, results, rule_cache, - rules_config, skills, + builtin_catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, + results, rule_cache, rules_config, skills, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -14,6 +14,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { custom_rule_evidence::register(module)?; rule_cache::register(module)?; custom_rule_cache::register(module)?; + builtin_catalogue::register(module)?; rules_config::register(module)?; results::register(module)?; skills::register(module) diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index bbb7d6c2f7..f2b6b08109 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -32,3 +32,4 @@ pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; pub(crate) const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v1"; +pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 48117ae5f2..5b57a1a7d4 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -850,12 +850,6 @@ impl fensu_policy::policy::types::PolicyRule for RuleMetadata { } } -#[derive(Debug, Serialize)] -pub struct CatalogueResponse { - pub version: u32, - pub rules: Vec, -} - /// One compiled model's facts as the rules request builder receives them. #[derive(Clone, Debug, Default)] pub struct ModelRow { diff --git a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs new file mode 100644 index 0000000000..f2ad474e0c --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs @@ -0,0 +1,53 @@ +//! `builtin_options`: the options each built-in rule declares. + +use crate::rules::models::{BuiltinOption, BuiltinOptionKind}; + +const fn boolean(name: &'static str, description: &'static str) -> BuiltinOption { + BuiltinOption { + kind: BuiltinOptionKind::Boolean, + name, + description, + } +} + +/// The declared options of the built-in rule `code`, in declaration order. +#[must_use] +pub fn builtin_options(code: &str) -> Vec { + match code { + "SQBRCONTRACT102" => vec![boolean( + "allow_numeric_indicators", + "Accept numeric is_/has_/can_ indicator columns.", + )], + "SQBRCONTRACT103" => vec![ + boolean( + "allow_date_for_at", + "Accept DATE for columns ending in _at.", + ), + boolean( + "allow_numeric_epoch", + "Accept numeric epoch values for timestamp-named columns.", + ), + boolean( + "allow_encoded_values", + "Accept string or variant encoded timestamp values.", + ), + ], + "SQBRCONTRACT104" => vec![ + boolean( + "allow_timestamps", + "Accept timestamp types for columns ending in _date.", + ), + boolean( + "allow_encoded_values", + "Accept string or variant encoded date values.", + ), + ], + "SQBRTEST203" => vec![BuiltinOption { + kind: BuiltinOptionKind::StringList, + name: "allowed_tests", + description: "Reviewed SQL test names that may assert only that empty inputs produce \ + no rows; listed tests count toward min_tests_per_model.", + }], + _ => Vec::new(), + } +} diff --git a/crates/sqlbuild-rules/src/rules/main/mod.rs b/crates/sqlbuild-rules/src/rules/main/mod.rs index bde8969bbb..a877872374 100644 --- a/crates/sqlbuild-rules/src/rules/main/mod.rs +++ b/crates/sqlbuild-rules/src/rules/main/mod.rs @@ -1,4 +1,5 @@ pub mod assemble_catalogue; +pub mod builtin_options; pub mod catalogue; pub mod evaluate; pub mod evaluate_project; @@ -6,4 +7,5 @@ pub mod fingerprint; pub mod resolve_threshold_overrides; pub mod select; pub mod selected_codes; +pub mod selected_sql_codes; pub mod sql_test_coverage; diff --git a/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs new file mode 100644 index 0000000000..c9778e61d1 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs @@ -0,0 +1,29 @@ +//! `selected_sql_codes`: the built-in SQL rule codes a `select`/`ignore` pair enables. + +use crate::constants::SQL_RULE_PREFIX; +use crate::rules::main::catalogue::catalogue; + +fn matches_any(code: &str, selectors: &[String]) -> bool { + for selector in selectors { + if code.starts_with(selector.as_str()) { + return true; + } + } + false +} + +/// Catalogue SQL rule codes matching a `select` prefix and no `ignore` prefix, in catalogue order. +#[must_use] +pub fn selected_sql_codes(select: &[String], ignore: &[String]) -> Vec { + let mut codes: Vec = Vec::new(); + for rule in catalogue() { + let code: String = rule.code; + if code.starts_with(SQL_RULE_PREFIX) + && matches_any(&code, select) + && !matches_any(&code, ignore) + { + codes.push(code); + } + } + codes +} diff --git a/crates/sqlbuild-rules/src/rules/models.rs b/crates/sqlbuild-rules/src/rules/models.rs index 969da8bdd7..0d9c6ec2a9 100644 --- a/crates/sqlbuild-rules/src/rules/models.rs +++ b/crates/sqlbuild-rules/src/rules/models.rs @@ -49,3 +49,20 @@ impl FaultCollector { self.faults.into_inner() } } + +/// The value kind of one built-in rule option. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BuiltinOptionKind { + /// A boolean defaulting to false. + Boolean, + /// A list of strings defaulting to empty. + StringList, +} + +/// One option a built-in rule declares. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BuiltinOption { + pub kind: BuiltinOptionKind, + pub name: &'static str, + pub description: &'static str, +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 90a8a3f391..5505a95cf1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -254,6 +254,7 @@ from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult +from sqlbuild.rule_engine.types import RuleCheck from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import ( @@ -307,7 +308,8 @@ class LintPreparationRequest(TypedDict): dialect: str def analyze_column_references_json(request_json: str) -> str: ... -def catalogue_json() -> str: ... +def builtin_rules(*, check: RuleCheck) -> list[Rule]: ... +def selected_sql_rule_codes(*, select: list[str], ignore: list[str]) -> list[str]: ... class NativeRulesRequest: """A natively built rules request held until it is evaluated once.""" diff --git a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py index 1ece17e899..852339009f 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py @@ -9,18 +9,14 @@ from importlib.machinery import ModuleSpec from pathlib import Path from types import ModuleType -from typing import cast +from sqlbuild._native import builtin_rules from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine._helpers.engine.native import native_catalogue, native_selected_codes +from sqlbuild.rule_engine._helpers.engine.native import native_selected_codes from sqlbuild.rule_engine.constants import ( - CONTRACT_BOOLEAN_NAME_CODE, - CONTRACT_DATE_NAME_CODE, - CONTRACT_TIMESTAMP_NAME_CODE, - EMPTY_INPUT_TEST_CODE, INIT_MODULE_NAME, RULE_DECORATOR_TOKEN, ) @@ -28,11 +24,9 @@ from sqlbuild.rule_engine.models import ( Finding, Rule, - RuleGuidance, RuleOption, RulesConfig, ) -from sqlbuild.rule_engine.types import RuleSubject def build_catalogue( @@ -65,107 +59,7 @@ def build_catalogue( def _builtins() -> tuple[Rule, ...]: - return tuple( - Rule( - code=str(item["code"]), - family=str(item["code"])[:-3], - slug=str(item["slug"]), - message=str(item["message"]), - remediation=str(item["remediation"]), - check=_native_builtin, - options=_builtin_options(str(item["code"])), - enabled_by_default=bool(item["enabled_by_default"]), - subject=(RuleSubject.PROJECT if bool(item["project_wide"]) else RuleSubject.MODEL), - guidance=_guidance(item.get("guidance")), - ) - for item in native_catalogue() - ) - - -def _builtin_options(code: str) -> tuple[RuleOption[object], ...]: - if code == CONTRACT_BOOLEAN_NAME_CODE: - return ( - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_numeric_indicators", - default=False, - description="Accept numeric is_/has_/can_ indicator columns.", - ), - ), - ) - if code == CONTRACT_TIMESTAMP_NAME_CODE: - return ( - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_date_for_at", - default=False, - description="Accept DATE for columns ending in _at.", - ), - ), - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_numeric_epoch", - default=False, - description="Accept numeric epoch values for timestamp-named columns.", - ), - ), - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_encoded_values", - default=False, - description="Accept string or variant encoded timestamp values.", - ), - ), - ) - if code == CONTRACT_DATE_NAME_CODE: - return ( - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_timestamps", - default=False, - description="Accept timestamp types for columns ending in _date.", - ), - ), - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_encoded_values", - default=False, - description="Accept string or variant encoded date values.", - ), - ), - ) - if code == EMPTY_INPUT_TEST_CODE: - return ( - cast( - RuleOption[object], - RuleOption.string_list( - name="allowed_tests", - default=(), - description=( - "Reviewed SQL test names that may assert only that empty inputs produce " - "no rows; listed tests count toward min_tests_per_model." - ), - ), - ), - ) - return () - - -def _guidance(value: object) -> RuleGuidance | None: - if not isinstance(value, dict): - return None - payload: dict[str, object] = {str(key): item for key, item in value.items()} - return RuleGuidance( - good_example=str(payload["good_example"]), - anti_tautology=str(payload["anti_tautology"]), - mutation_check=str(payload["mutation_check"]), - ) + return tuple(builtin_rules(check=_native_builtin)) def _native_builtin(*, model: object, ctx: object) -> list[Finding]: diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 604eaa9c9d..b1c6147abe 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -44,24 +44,6 @@ def finalize_native_findings( ) -def native_catalogue() -> tuple[dict[str, object], ...]: - """Return native-owned built-in rule metadata.""" - - try: - payload: object = json.loads(_native.catalogue_json()) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - if not isinstance(payload, dict) or not isinstance(payload.get("rules"), list): - raise RulesError("native rules engine returned an invalid catalogue") - rules: list[object] = payload["rules"] - if any(not isinstance(value, dict) for value in rules): - raise RulesError("native rules engine returned invalid rule metadata") - normalized: list[dict[str, object]] = [] - for value in rules: - normalized.append({str(key): item for key, item in value.items()}) - return tuple(normalized) - - def native_selected_codes( *, config: RulesConfig, diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 9f5edda053..9d08af870c 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -12,6 +12,7 @@ load_sql_rule_cache, lookup_sql_rule_cache, project_with, + selected_sql_rule_codes, sql_file_rule_key, sql_model_rule_keys, sql_rule_lint_key, @@ -54,7 +55,6 @@ from sqlbuild.rule_engine._helpers.engine.native import ( custom_rule_payloads, finalize_native_findings, - native_catalogue, ) from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings @@ -339,16 +339,7 @@ def _selection_rules_config( def _selected_sql_codes(config: RulesConfig) -> tuple[str, ...]: - codes: list[str] = [] - for rule in native_catalogue(): - code: str = str(rule["code"]) - if not code.startswith("SQBRSQL"): - continue - selected: bool = any(code.startswith(selector) for selector in config.select) - ignored: bool = any(code.startswith(selector) for selector in config.ignore) - if selected and not ignored: - codes.append(code) - return tuple(codes) + return tuple(selected_sql_rule_codes(select=list(config.select), ignore=list(config.ignore))) def _selected_project( diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index 580c8d57a4..d985125218 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -79,10 +79,6 @@ AST_SELECT_KIND: str = "select" AST_CTE_KIND: str = "cte" CONTRACT_ENFORCED: str = "enforced" -CONTRACT_BOOLEAN_NAME_CODE: str = "SQBRCONTRACT102" -CONTRACT_TIMESTAMP_NAME_CODE: str = "SQBRCONTRACT103" -CONTRACT_DATE_NAME_CODE: str = "SQBRCONTRACT104" -EMPTY_INPUT_TEST_CODE: str = "SQBRTEST203" TYPE_PROOF_RULE_CODES: frozenset[str] = frozenset({"SQBRCONTRACT105"}) BOOLEAN_TYPE: str = "BOOLEAN" From f3a7da07bc3c34eaa7dbec65375d3fe3ef369e0e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:34:11 +0100 Subject: [PATCH 32/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 40592263c0..fe3c81c39d 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 82303 +remaining_lines = 81100 [[stays]] path = "executor" From cb88b0450084cc9c1689cfadd95f6d4ab93a07fc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:44:34 +0100 Subject: [PATCH 33/37] refactor(cli): run sqb diff validation, orchestration and query lifecycle natively --- .../src/diff_report/_helpers/mod.rs | 3 + .../src/diff_report/_helpers/request_rules.rs | 209 +++++ .../src/diff_report/_helpers/shell_words.rs | 16 + .../src/diff_report/_helpers/size_guard.rs | 84 ++ .../sqlbuild-cli/src/diff_report/constants.rs | 32 + crates/sqlbuild-cli/src/diff_report/errors.rs | 9 + .../main/effective_max_examples.rs | 13 + .../main/is_incomplete_query_failure.rs | 13 + .../diff_report/main/is_query_diff_request.rs | 9 + .../sqlbuild-cli/src/diff_report/main/mod.rs | 8 + .../diff_report/main/parse_diff_name_range.rs | 18 + .../diff_report/main/query_column_problem.rs | 56 ++ .../main/read_only_query_problem.rs | 33 + .../src/diff_report/main/size_guard_report.rs | 94 +++ .../diff_report/main/validate_diff_request.rs | 20 + crates/sqlbuild-cli/src/diff_report/mod.rs | 1 + crates/sqlbuild-cli/src/diff_report/models.rs | 73 ++ crates/sqlbuild-cli/src/diff_report/tests.rs | 2 + .../src/diff_report/tests/helpers.rs | 37 +- .../src/diff_report/tests/test_command.rs | 250 ++++++ .../src/diff_report/tests/test_types.rs | 34 +- crates/sqlbuild-cli/src/diff_report/types.rs | 10 + crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- .../src/bindings/_helpers/cli/dispatch.rs | 17 +- .../_helpers/command_runtime/warehouses.rs | 2 +- .../bindings/_helpers/diff_report/command.rs | 524 ++++++++++++ .../src/bindings/_helpers/diff_report/mod.rs | 3 + .../bindings/_helpers/diff_report/query.rs | 772 ++++++++++++++++++ .../_helpers/diff_report/registration.rs | 94 +-- .../bindings/_helpers/diff_report/support.rs | 160 ++++ src/sqlbuild/_native.pyi | 29 +- .../cli/commands/_helpers/diff/__init__.py | 1 - .../cli/commands/_helpers/diff/execution.py | 685 ---------------- .../cli/commands/_helpers/diff/invocation.py | 164 ---- .../cli/commands/_helpers/diff/size_guard.py | 193 ----- .../cli/commands/_helpers/diff/validation.py | 19 - src/sqlbuild/cli/commands/constants.py | 7 - .../cli/commands/main/inspection/_diff.py | 139 ---- src/sqlbuild/cli/commands/models.py | 37 +- ...pace.py => _effective_target_namespace.py} | 0 ...guration.py => _discover_configuration.py} | 0 src/sqlbuild/executor/diff/constants.py | 2 - .../diff/main/{config.py => _config.py} | 0 .../{execute_query.py => _execute_query.py} | 0 .../diff/test_query_artifact_cleanup.py | 2 +- .../cli/commands/_helpers/diff/_test_types.py | 42 +- .../cli/commands/_helpers/diff/helpers.py | 86 -- .../commands/_helpers/diff/test_size_guard.py | 186 ----- .../commands/_helpers/diff/test_validation.py | 68 -- 49 files changed, 2526 insertions(+), 1734 deletions(-) create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/request_rules.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/shell_words.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/_helpers/size_guard.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/errors.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/effective_max_examples.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/is_incomplete_query_failure.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/is_query_diff_request.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/parse_diff_name_range.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/query_column_problem.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/read_only_query_problem.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/size_guard_report.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/main/validate_diff_request.rs create mode 100644 crates/sqlbuild-cli/src/diff_report/tests/test_command.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/diff/__init__.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/diff/execution.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/diff/invocation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/diff/size_guard.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/diff/validation.py delete mode 100644 src/sqlbuild/cli/commands/main/inspection/_diff.py rename src/sqlbuild/compiler/compile/main/{effective_target_namespace.py => _effective_target_namespace.py} (100%) rename src/sqlbuild/compiler/discovery/main/{discover_configuration.py => _discover_configuration.py} (100%) rename src/sqlbuild/executor/diff/main/{config.py => _config.py} (100%) rename src/sqlbuild/executor/diff/main/{execute_query.py => _execute_query.py} (100%) delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/diff/helpers.py delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_size_guard.py delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_validation.py diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs index f847df459f..7a2971b107 100644 --- a/crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs @@ -3,5 +3,8 @@ pub(crate) mod findings; pub(crate) mod layout; pub(crate) mod numbers; pub(crate) mod payload; +pub(crate) mod request_rules; pub(crate) mod sections; +pub(crate) mod shell_words; +pub(crate) mod size_guard; pub(crate) mod text_cells; diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/request_rules.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/request_rules.rs new file mode 100644 index 0000000000..489ddadb55 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/request_rules.rs @@ -0,0 +1,209 @@ +//! The individual `sqb diff` flag rules. + +use crate::diff_report::errors::DiffUserError; +use crate::diff_report::models::DiffRequestFacts; + +pub(crate) fn user_error(message: &str, code: &str) -> DiffUserError { + DiffUserError { + message: message.to_owned(), + code: code.to_owned(), + help: None, + } +} + +fn not_positive(value: Option) -> bool { + value.is_some_and(|value| value <= 0) +} + +fn blank(value: Option<&String>) -> bool { + value.is_some_and(|value| value.trim().is_empty()) +} + +/// The first violated rule that applies to model and raw-query diffs alike. +pub(crate) fn common_violation(request: &DiffRequestFacts) -> Option<(&'static str, &'static str)> { + let modes: usize = usize::from(request.full) + + usize::from(request.schema_only) + + usize::from(request.bounded.is_some()); + let mut lowered: Vec = request + .unique_key_override + .iter() + .map(|key| key.to_lowercase()) + .collect(); + lowered.sort(); + let total: usize = lowered.len(); + lowered.dedup(); + let rules: [(bool, &str, &str); 15] = [ + ( + modes > 1, + "diff accepts at most one of --full, --schema-only, or --bounded", + "C201", + ), + ( + not_positive(request.max_column_examples), + "diff --max-column-examples must be positive", + "C202", + ), + ( + not_positive(request.max_row_only_examples), + "diff --max-row-only-examples must be positive", + "C203", + ), + ( + not_positive(request.sample_rows), + "diff --sample-rows must be positive", + "C208", + ), + ( + request.exhaustive && request.sample_rows.is_some(), + "diff --exhaustive cannot be combined with --sample-rows", + "C209", + ), + ( + not_positive(request.max_models), + "diff --max-models must be positive", + "C210", + ), + ( + not_positive(request.max_columns), + "diff --max-columns must be positive", + "C211", + ), + ( + not_positive(request.max_value_length), + "diff --max-value-length must be positive", + "C246", + ), + ( + request.full_example_values && request.max_value_length.is_some(), + "diff --full-example-values cannot be combined with --max-value-length", + "C247", + ), + ( + request.full_example_values && request.suppress_example_values, + "diff --full-example-values cannot be combined with --no-example-values", + "C248", + ), + ( + request.unkeyed && !request.unique_key_override.is_empty(), + "diff --unkeyed cannot be combined with --key", + "C226", + ), + ( + request.unkeyed && (request.sample_rows.is_some() || request.sample_seed.is_some()), + "unkeyed diff does not support sampling", + "C222", + ), + ( + request.unkeyed && !request.tolerance_overrides.is_empty(), + "unkeyed diff does not support tolerances", + "C223", + ), + ( + request + .unique_key_override + .iter() + .any(|key| key.trim().is_empty()), + "diff --key values must not be empty", + "C241", + ), + ( + lowered.len() != total, + "diff --key values must be unique", + "C242", + ), + ]; + rules + .into_iter() + .find(|(violated, _, _)| *violated) + .map(|(_, message, code)| (message, code)) +} + +/// The first violated rule specific to raw-query diffs. +pub(crate) fn query_violation(request: &DiffRequestFacts) -> Option<(&'static str, &'static str)> { + let left: bool = request.left_query || request.left_query_file; + let right: bool = request.right_query || request.right_query_file; + let keyed: bool = !request.unique_key_override.is_empty(); + let rules: [(bool, &str, &str); 12] = [ + ( + request.from_name.is_some() || request.to_name.is_some(), + "raw-query diff does not accept FROM:TO", + "C212", + ), + ( + !request.select.is_empty() || !request.exclude.is_empty(), + "raw-query diff does not accept model selectors", + "C213", + ), + ( + request.bounded.is_some(), + "raw-query diff does not support --bounded; filter both raw queries explicitly", + "C214", + ), + ( + request.max_models.is_some(), + "raw-query diff does not accept --max-models", + "C243", + ), + ( + blank(request.left_label.as_ref()), + "raw-query diff --left-label must not be empty", + "C249", + ), + ( + blank(request.right_label.as_ref()), + "raw-query diff --right-label must not be empty", + "C250", + ), + (!left, "raw-query diff requires a left query", "C216"), + ( + request.left_query && request.left_query_file, + "raw-query diff accepts either --left-query or --left-query-file, not both", + "C217", + ), + (!right, "raw-query diff requires a right query", "C218"), + ( + request.right_query && request.right_query_file, + "raw-query diff accepts either --right-query or --right-query-file, not both", + "C219", + ), + ( + request.schema_only && (request.unkeyed || keyed), + "schema-only raw-query diff does not accept --key or --unkeyed", + "C220", + ), + ( + !request.schema_only && keyed == request.unkeyed, + "raw-query diff requires one or more --key values or explicit --unkeyed", + "C221", + ), + ]; + rules + .into_iter() + .find(|(violated, _, _)| *violated) + .map(|(_, message, code)| (message, code)) +} + +/// The first violated rule specific to model diffs. +pub(crate) fn model_violation(request: &DiffRequestFacts) -> Option<(&'static str, &'static str)> { + let rules: [(bool, &str, &str); 3] = [ + ( + request.from_name.is_none(), + "model diff requires FROM or FROM:TO", + "C224", + ), + ( + request.selected_target.is_some(), + "model diff does not accept --target", + "C225", + ), + ( + request.left_label.is_some() || request.right_label.is_some(), + "model diff does not accept query labels", + "C251", + ), + ]; + rules + .into_iter() + .find(|(violated, _, _)| *violated) + .map(|(_, message, code)| (message, code)) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/shell_words.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/shell_words.rs new file mode 100644 index 0000000000..dfc11f62df --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/shell_words.rs @@ -0,0 +1,16 @@ +//! Python's `shlex.quote`. + +fn is_safe(character: char) -> bool { + character.is_ascii_alphanumeric() || "_@%+=:,./-".contains(character) +} + +/// One shell word, quoted only when it holds characters outside the safe set. +pub(crate) fn quote(word: &str) -> String { + if word.is_empty() { + return "''".to_owned(); + } + if word.chars().all(is_safe) { + return word.to_owned(); + } + format!("'{}'", word.replace('\'', "'\"'\"'")) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/size_guard.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/size_guard.rs new file mode 100644 index 0000000000..bc3feae768 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/size_guard.rs @@ -0,0 +1,84 @@ +//! Sizes, payloads and replacement commands of the full-diff size guard. + +use sqlbuild_core::json::models::JsonValue; + +use crate::diff_report::_helpers::numbers::grouped; +use crate::diff_report::_helpers::payload::{integer, object, text}; +use crate::diff_report::_helpers::shell_words::quote; +use crate::diff_report::constants::BOUNDED_WINDOW_PLACEHOLDER; +use crate::diff_report::models::{DiffRequestFacts, FullDiffSideFacts}; + +pub(crate) fn side_text(side: &FullDiffSideFacts) -> String { + let limit: String = side + .max_rows + .map_or_else(|| "unlimited".to_owned(), grouped); + let size: String = match (side.row_count, &side.detail) { + (Some(rows), _) => format!("{} rows", grouped(rows)), + (None, Some(detail)) => format!("size unknown: {detail}"), + (None, None) => "size unknown".to_owned(), + }; + format!("{} {size}, limit {limit}", side.target) +} + +pub(crate) fn optional_integer(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, integer) +} + +pub(crate) fn side_payload(side: &FullDiffSideFacts) -> JsonValue { + object(vec![ + ("target", text(&side.target)), + ("relation", text(&side.relation)), + ("row_count", optional_integer(side.row_count)), + ("max_full_rows", optional_integer(side.max_rows)), + ("exceeds_limit", JsonValue::Bool(side.exceeds_limit)), + ( + "detail", + side.detail.as_deref().map_or(JsonValue::Null, text), + ), + ]) +} + +pub(crate) fn scope_arguments(request: &DiffRequestFacts) -> Vec { + let mut arguments: Vec = Vec::new(); + for (flag, values) in [ + ("--select", &request.select), + ("--exclude", &request.exclude), + ("--key", &request.unique_key_override), + ] { + if !values.is_empty() { + arguments.push(flag.to_owned()); + arguments.extend(values.iter().cloned()); + } + } + if request.unkeyed { + arguments.push("--unkeyed".to_owned()); + } + if !request.excluded_columns_override.is_empty() { + arguments.push("--exclude-column".to_owned()); + arguments.extend(request.excluded_columns_override.iter().cloned()); + } + for tolerance in &request.tolerance_overrides { + arguments.extend(["--tolerance".to_owned(), tolerance.clone()]); + } + if let Some(variables) = &request.cli_vars_json { + arguments.extend(["--vars".to_owned(), variables.clone()]); + } + arguments +} + +pub(crate) fn command(prefix: &[String], mode: &[&str], scope: &[String]) -> String { + prefix + .iter() + .map(String::as_str) + .chain(mode.iter().copied()) + .chain(scope.iter().map(String::as_str)) + .map(|part| { + if part == BOUNDED_WINDOW_PLACEHOLDER { + part.to_owned() + } else { + quote(part) + } + }) + .collect::>() + .join(" ") +} diff --git a/crates/sqlbuild-cli/src/diff_report/constants.rs b/crates/sqlbuild-cli/src/diff_report/constants.rs index a67d7a65b3..bda83c40ea 100644 --- a/crates/sqlbuild-cli/src/diff_report/constants.rs +++ b/crates/sqlbuild-cli/src/diff_report/constants.rs @@ -8,3 +8,35 @@ pub(crate) const QUERY_INPUT_KIND: &str = "query"; pub(crate) const FINDINGS_OUTCOME: &str = "findings"; pub(crate) const PASS_OUTCOME: &str = "pass"; pub(crate) const INCOMPLETE_OUTCOME: &str = "incomplete"; +pub(crate) const BOUNDED_WINDOW_PLACEHOLDER: &str = ""; +/// Root expression kinds a raw diff query may have. +pub(crate) const READ_ONLY_QUERY_ROOT_KEYS: [&str; 5] = + ["except", "intersect", "select", "union", "values"]; +/// Expression kinds that make a raw diff query change data or state. +pub(crate) const MUTATING_QUERY_KEYS: [&str; 19] = [ + "alter", + "command", + "copy", + "create", + "delete", + "drop", + "execute", + "grant", + "insert", + "load_data", + "merge", + "pragma", + "revoke", + "set", + "transaction", + "truncate", + "uncache", + "update", + "use", +]; +pub(crate) const INCOMPLETE_PREPARATION_CODES: [&str; 9] = [ + "C229", "C230", "C231", "C232", "C233", "C234", "C235", "C240", "C244", +]; +pub(crate) const INCOMPLETE_EXECUTION_CODES: [&str; 3] = ["C237", "C238", "C239"]; +pub(crate) const DEFAULT_EXAMPLES: i64 = 3; +pub(crate) const VERBOSE_EXAMPLES: i64 = 10; diff --git a/crates/sqlbuild-cli/src/diff_report/errors.rs b/crates/sqlbuild-cli/src/diff_report/errors.rs new file mode 100644 index 0000000000..c0c8c8b9b5 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/errors.rs @@ -0,0 +1,9 @@ +//! Rejections of `sqb diff` requests and inputs. + +/// A rejected diff request or input: message, `C` code and optional help. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DiffUserError { + pub message: String, + pub code: String, + pub help: Option, +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/effective_max_examples.rs b/crates/sqlbuild-cli/src/diff_report/main/effective_max_examples.rs new file mode 100644 index 0000000000..fe916f2b34 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/effective_max_examples.rs @@ -0,0 +1,13 @@ +//! How many examples a diff shows per column or side when not set explicitly. + +use crate::diff_report::constants::{DEFAULT_EXAMPLES, VERBOSE_EXAMPLES}; + +/// The explicit limit, else ten when verbose and three otherwise. +#[must_use] +pub fn effective_max_examples(explicit: Option, verbose: bool) -> i64 { + explicit.unwrap_or(if verbose { + VERBOSE_EXAMPLES + } else { + DEFAULT_EXAMPLES + }) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/is_incomplete_query_failure.rs b/crates/sqlbuild-cli/src/diff_report/main/is_incomplete_query_failure.rs new file mode 100644 index 0000000000..c63c7fb723 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/is_incomplete_query_failure.rs @@ -0,0 +1,13 @@ +//! Whether a raw-query diff user error means incomplete evidence rather than a failed run. + +use crate::diff_report::constants::{INCOMPLETE_EXECUTION_CODES, INCOMPLETE_PREPARATION_CODES}; +use crate::diff_report::types::QueryDiffPhase; + +/// Input and preflight rejections are incomplete (exit 2); other failures are execution failures. +#[must_use] +pub fn is_incomplete_query_failure(phase: QueryDiffPhase, code: &str) -> bool { + match phase { + QueryDiffPhase::Preparation => INCOMPLETE_PREPARATION_CODES.contains(&code), + QueryDiffPhase::Execution => INCOMPLETE_EXECUTION_CODES.contains(&code), + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/is_query_diff_request.rs b/crates/sqlbuild-cli/src/diff_report/main/is_query_diff_request.rs new file mode 100644 index 0000000000..43bb8fcec5 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/is_query_diff_request.rs @@ -0,0 +1,9 @@ +//! Whether a diff compares raw queries instead of project models. + +use crate::diff_report::models::DiffRequestFacts; + +/// Any raw-query input was supplied. +#[must_use] +pub fn is_query_diff_request(request: &DiffRequestFacts) -> bool { + request.left_query || request.left_query_file || request.right_query || request.right_query_file +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/mod.rs b/crates/sqlbuild-cli/src/diff_report/main/mod.rs index 86714c78a4..4535034e41 100644 --- a/crates/sqlbuild-cli/src/diff_report/main/mod.rs +++ b/crates/sqlbuild-cli/src/diff_report/main/mod.rs @@ -1,4 +1,12 @@ pub mod diff_exit_code; +pub mod effective_max_examples; +pub mod is_incomplete_query_failure; +pub mod is_query_diff_request; +pub mod parse_diff_name_range; +pub mod query_column_problem; +pub mod read_only_query_problem; pub mod render_diff_error_json; pub mod render_diff_json; pub mod render_diff_text; +pub mod size_guard_report; +pub mod validate_diff_request; diff --git a/crates/sqlbuild-cli/src/diff_report/main/parse_diff_name_range.rs b/crates/sqlbuild-cli/src/diff_report/main/parse_diff_name_range.rs new file mode 100644 index 0000000000..791703b50a --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/parse_diff_name_range.rs @@ -0,0 +1,18 @@ +//! The positional `FROM[:TO]` target range of `sqb diff`. + +use crate::diff_report::_helpers::request_rules::user_error; +use crate::diff_report::errors::DiffUserError; + +/// `FROM:TO` or a bare `FROM`, which leaves `TO` unset. +pub fn parse_diff_name_range(range: &str) -> Result<(String, Option), DiffUserError> { + let (from_name, to_name): (&str, Option<&str>) = match range.split_once(':') { + Some((from_name, to_name)) => (from_name, Some(to_name)), + None => (range, None), + }; + if from_name.is_empty() + || to_name.is_some_and(|to_name| to_name.is_empty() || to_name.contains(':')) + { + return Err(user_error("diff range must be FROM:TO or FROM", "C209")); + } + Ok((from_name.to_owned(), to_name.map(str::to_owned))) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/query_column_problem.rs b/crates/sqlbuild-cli/src/diff_report/main/query_column_problem.rs new file mode 100644 index 0000000000..dd2f1be65a --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/query_column_problem.rs @@ -0,0 +1,56 @@ +//! Preflight checks of the columns one raw diff query returns. + +use crate::diff_report::errors::DiffUserError; + +/// No columns (C237), duplicate names ignoring case (C238), or too many columns (C239). +pub fn query_column_problem( + label: &str, + columns: &[String], + max_columns: Option, +) -> Result<(), DiffUserError> { + let display: String = if label.to_lowercase().ends_with("query") { + label.to_owned() + } else { + format!("{label} query") + }; + if columns.is_empty() { + return Err(DiffUserError { + message: format!("{display} returned no columns"), + code: "C237".to_owned(), + help: None, + }); + } + let lowered: Vec = columns.iter().map(|column| column.to_lowercase()).collect(); + let mut sorted: Vec<&String> = lowered.iter().collect(); + sorted.sort(); + let mut duplicates: Vec<&String> = Vec::new(); + for pair in sorted.windows(2) { + if pair[0] == pair[1] { + duplicates.push(pair[0]); + } + } + duplicates.sort(); + duplicates.dedup(); + if !duplicates.is_empty() { + let names: Vec<&str> = duplicates.into_iter().map(String::as_str).collect(); + return Err(DiffUserError { + message: format!( + "{display} returns duplicate column names: {}", + names.join(", ") + ), + code: "C238".to_owned(), + help: Some("add unique aliases before comparing query results".to_owned()), + }); + } + match max_columns { + Some(limit) if columns.len() as i64 > limit => Err(DiffUserError { + message: format!( + "{display} has {} columns, exceeding --max-columns {limit}", + columns.len() + ), + code: "C239".to_owned(), + help: None, + }), + _ => Ok(()), + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/read_only_query_problem.rs b/crates/sqlbuild-cli/src/diff_report/main/read_only_query_problem.rs new file mode 100644 index 0000000000..f187afc9aa --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/read_only_query_problem.rs @@ -0,0 +1,33 @@ +//! Whether one parsed raw diff query is a single read-only expression. + +use crate::diff_report::_helpers::request_rules::user_error; +use crate::diff_report::constants::{MUTATING_QUERY_KEYS, READ_ONLY_QUERY_ROOT_KEYS}; +use crate::diff_report::errors::DiffUserError; +use crate::diff_report::models::QueryShapeFacts; + +/// A non-query root (C235), any data-changing node (C240), or an `INTO`/lock clause (C244). +pub fn read_only_query_problem(label: &str, shape: &QueryShapeFacts) -> Result<(), DiffUserError> { + if !READ_ONLY_QUERY_ROOT_KEYS.contains(&shape.root_key.as_str()) { + return Err(user_error( + &format!("{label} query must be one read-only SELECT, WITH, or VALUES expression"), + "C235", + )); + } + if shape + .node_keys + .iter() + .any(|key| MUTATING_QUERY_KEYS.contains(&key.as_str())) + { + return Err(user_error( + &format!("{label} query must not contain data-changing or administrative statements"), + "C240", + )); + } + if shape.writes_or_locks { + return Err(user_error( + &format!("{label} query must not write a relation or acquire row locks"), + "C244", + )); + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/size_guard_report.rs b/crates/sqlbuild-cli/src/diff_report/main/size_guard_report.rs new file mode 100644 index 0000000000..23427cb684 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/size_guard_report.rs @@ -0,0 +1,94 @@ +//! The C270 report of a default full diff stopped by table sizes. + +use sqlbuild_core::json::models::JsonValue; + +use crate::diff_report::_helpers::payload::{object, text}; +use crate::diff_report::_helpers::size_guard::{command, scope_arguments, side_payload, side_text}; +use crate::diff_report::constants::BOUNDED_WINDOW_PLACEHOLDER; +use crate::diff_report::models::{DiffRequestFacts, FullDiffModelFacts, SizeGuardReport}; + +/// Every blocked model, the explicit replacement commands, and how to raise the limits. +#[must_use] +pub fn size_guard_report( + request: &DiffRequestFacts, + (from_target, to_target): (&str, &str), + guard_message: &str, + blocked: &[FullDiffModelFacts], +) -> SizeGuardReport { + let scope: Vec = scope_arguments(request); + let mut prefix: Vec = vec!["sqb".to_owned()]; + if let Some(project_dir) = &request.project_dir { + prefix.extend(["--project-dir".to_owned(), project_dir.clone()]); + } + prefix.extend(["diff".to_owned(), format!("{from_target}:{to_target}")]); + let full: String = command(&prefix, &["--full"], &scope); + let bounded: Option = blocked + .iter() + .all(|model| model.has_cursor) + .then(|| command(&prefix, &["--bounded", BOUNDED_WINDOW_PLACEHOLDER], &scope)); + let schema_only: String = command(&prefix, &["--schema-only"], &scope); + let mut lines: Vec = vec![guard_message.to_owned()]; + for model in blocked { + lines.push(format!( + " {}: {}; {}{}", + model.name, + side_text(&model.left), + side_text(&model.right), + if model.has_cursor { "" } else { "; no cursor" } + )); + } + lines.push("Run one of these instead:".to_owned()); + lines.push(format!(" {full}")); + if let Some(bounded) = &bounded { + lines.push(format!(" {bounded}")); + } + lines.push(format!(" {schema_only}")); + let mut over_targets: Vec<&str> = Vec::new(); + for side in blocked.iter().flat_map(|model| [&model.left, &model.right]) { + if side.exceeds_limit && !over_targets.contains(&side.target.as_str()) { + over_targets.push(&side.target); + } + } + let bounded_help: String = if bounded.is_some() { + format!( + ", --bounded compares a recent cursor window (replace {BOUNDED_WINDOW_PLACEHOLDER} with a duration such as 7d)" + ) + } else { + String::new() + }; + let raise: Vec = over_targets + .iter() + .map(|target| format!("targets.{target}.diff.max_full_rows")) + .collect(); + let models: Vec = blocked + .iter() + .map(|model| { + object(vec![ + ("name", text(&model.name)), + ("has_cursor", JsonValue::Bool(model.has_cursor)), + ("from", side_payload(&model.left)), + ("to", side_payload(&model.right)), + ]) + }) + .collect(); + SizeGuardReport { + message: lines.join("\n"), + help: format!( + "--full compares every row{bounded_help}, and --schema-only compares columns only; or raise {}", + raise.join(" and ") + ), + details: object(vec![ + ("from_target", text(from_target)), + ("to_target", text(to_target)), + ("models", JsonValue::Array(models)), + ( + "commands", + object(vec![ + ("full", text(&full)), + ("bounded", bounded.as_deref().map_or(JsonValue::Null, text)), + ("schema_only", text(&schema_only)), + ]), + ), + ]), + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/validate_diff_request.rs b/crates/sqlbuild-cli/src/diff_report/main/validate_diff_request.rs new file mode 100644 index 0000000000..37e1eb05d7 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/validate_diff_request.rs @@ -0,0 +1,20 @@ +//! Reject contradictory or out-of-range `sqb diff` flags before any discovery. + +use crate::diff_report::_helpers::request_rules::{ + common_violation, model_violation, query_violation, user_error, +}; +use crate::diff_report::errors::DiffUserError; +use crate::diff_report::main::is_query_diff_request::is_query_diff_request; +use crate::diff_report::models::DiffRequestFacts; + +/// The first violated rule, checked in the order users see them reported. +pub fn validate_diff_request(request: &DiffRequestFacts) -> Result<(), DiffUserError> { + let violation: Option<(&str, &str)> = common_violation(request).or_else(|| { + if is_query_diff_request(request) { + query_violation(request) + } else { + model_violation(request) + } + }); + violation.map_or(Ok(()), |(message, code)| Err(user_error(message, code))) +} diff --git a/crates/sqlbuild-cli/src/diff_report/mod.rs b/crates/sqlbuild-cli/src/diff_report/mod.rs index 5f9d1eb986..50aec51473 100644 --- a/crates/sqlbuild-cli/src/diff_report/mod.rs +++ b/crates/sqlbuild-cli/src/diff_report/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod _helpers; pub(crate) mod constants; +pub mod errors; pub mod main; pub mod models; pub mod types; diff --git a/crates/sqlbuild-cli/src/diff_report/models.rs b/crates/sqlbuild-cli/src/diff_report/models.rs index 10ddab3fd8..50ed8eb30c 100644 --- a/crates/sqlbuild-cli/src/diff_report/models.rs +++ b/crates/sqlbuild-cli/src/diff_report/models.rs @@ -130,3 +130,76 @@ pub(crate) struct RenderedValue { pub(crate) suppressed: bool, pub(crate) first_difference: Option, } + +/// The `sqb diff` flags that request validation reads. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DiffRequestFacts { + pub from_name: Option, + pub to_name: Option, + pub full: bool, + pub schema_only: bool, + pub bounded: Option, + pub max_column_examples: Option, + pub max_row_only_examples: Option, + pub sample_rows: Option, + pub sample_seed: Option, + pub exhaustive: bool, + pub max_models: Option, + pub max_columns: Option, + pub max_value_length: Option, + pub full_example_values: bool, + pub suppress_example_values: bool, + pub select: Vec, + pub exclude: Vec, + pub selected_target: Option, + pub left_query: bool, + pub left_query_file: bool, + pub right_query: bool, + pub right_query_file: bool, + pub left_label: Option, + pub right_label: Option, + pub unique_key_override: Vec, + pub unkeyed: bool, + pub excluded_columns_override: Vec, + pub tolerance_overrides: Vec, + /// `json.dumps(cli_vars, sort_keys=True)` when variables were passed. + pub cli_vars_json: Option, + pub project_dir: Option, +} + +/// One side of a model the default full diff refused to compare. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FullDiffSideFacts { + pub target: String, + pub relation: String, + pub row_count: Option, + pub max_rows: Option, + pub exceeds_limit: bool, + pub detail: Option, +} + +/// One model blocked by the default full diff size guard. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FullDiffModelFacts { + pub name: String, + pub has_cursor: bool, + pub left: FullDiffSideFacts, + pub right: FullDiffSideFacts, +} + +/// The C270 size-guard error: message, help and machine-readable details. +#[derive(Clone, Debug, PartialEq)] +pub struct SizeGuardReport { + pub message: String, + pub help: String, + pub details: JsonValue, +} + +/// The parsed shape of one raw diff query: expression kinds, lowercased, and write clauses. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct QueryShapeFacts { + pub root_key: String, + pub node_keys: Vec, + /// The root expression has an `INTO` target or row-locking clauses. + pub writes_or_locks: bool, +} diff --git a/crates/sqlbuild-cli/src/diff_report/tests.rs b/crates/sqlbuild-cli/src/diff_report/tests.rs index d2b8e61c63..eed637fb30 100644 --- a/crates/sqlbuild-cli/src/diff_report/tests.rs +++ b/crates/sqlbuild-cli/src/diff_report/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_command.rs"] +mod command; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_render.rs"] diff --git a/crates/sqlbuild-cli/src/diff_report/tests/helpers.rs b/crates/sqlbuild-cli/src/diff_report/tests/helpers.rs index 7c8a9c812e..150fe5f99f 100644 --- a/crates/sqlbuild-cli/src/diff_report/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/diff_report/tests/helpers.rs @@ -1,8 +1,8 @@ use sqlbuild_core::json::models::JsonValue; use crate::diff_report::models::{ - DiffColumnFacts, DiffDocumentOptions, DiffExampleOptions, DiffModelFacts, DiffRowFacts, - DiffValue, + DiffColumnFacts, DiffDocumentOptions, DiffExampleOptions, DiffModelFacts, DiffRequestFacts, + DiffRowFacts, DiffValue, FullDiffModelFacts, FullDiffSideFacts, }; pub(crate) fn changed_rows(unequal: i64) -> DiffRowFacts { @@ -62,3 +62,36 @@ pub(crate) fn cursor_rows() -> DiffRowFacts { ..changed_rows(1) } } + +pub(crate) fn request(build: impl FnOnce(&mut DiffRequestFacts)) -> DiffRequestFacts { + let mut facts: DiffRequestFacts = DiffRequestFacts::default(); + build(&mut facts); + facts +} + +pub(crate) fn blocked_orders(has_cursor: bool) -> Vec { + vec![FullDiffModelFacts { + name: "orders".to_owned(), + has_cursor, + left: FullDiffSideFacts { + target: "prod".to_owned(), + relation: "prod.orders".to_owned(), + row_count: Some(12_500_000), + max_rows: Some(10_000_000), + exceeds_limit: true, + detail: None, + }, + right: FullDiffSideFacts { + target: "dev".to_owned(), + relation: "dev.orders".to_owned(), + row_count: None, + max_rows: Some(10_000_000), + exceeds_limit: true, + detail: Some("no row count (view)".to_owned()), + }, + }] +} + +pub(crate) fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} diff --git a/crates/sqlbuild-cli/src/diff_report/tests/test_command.rs b/crates/sqlbuild-cli/src/diff_report/tests/test_command.rs new file mode 100644 index 0000000000..ed284add8f --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/tests/test_command.rs @@ -0,0 +1,250 @@ +use crate::diff_report::main::parse_diff_name_range::parse_diff_name_range; +use crate::diff_report::main::query_column_problem::query_column_problem; +use crate::diff_report::main::size_guard_report::size_guard_report; +use crate::diff_report::main::validate_diff_request::validate_diff_request; +use crate::diff_report::tests::helpers::{blocked_orders, request, strings}; +use crate::diff_report::tests::test_types::{ + ColumnsTestCase, NameRangeTestCase, RequestTestCase, SizeGuardTestCase, +}; + +#[test] +fn given_target_range_when_parsing_then_targets_or_code_match() { + let parsed = |from: &str, to: Option<&str>| Ok((from.to_owned(), to.map(str::to_owned))); + let test_cases = [ + NameRangeTestCase { + description: "explicit range keeps both targets", + range: "prod:dev", + expected_targets: parsed("prod", Some("dev")), + }, + NameRangeTestCase { + description: "bare target leaves TO unset", + range: "prod", + expected_targets: parsed("prod", None), + }, + NameRangeTestCase { + description: "empty target", + range: "", + expected_targets: Err("C209".to_owned()), + }, + NameRangeTestCase { + description: "empty TO", + range: "prod:", + expected_targets: Err("C209".to_owned()), + }, + NameRangeTestCase { + description: "empty FROM", + range: ":dev", + expected_targets: Err("C209".to_owned()), + }, + NameRangeTestCase { + description: "too many separators", + range: "a:b:c", + expected_targets: Err("C209".to_owned()), + }, + ]; + + for test_case in test_cases { + let parsed = parse_diff_name_range(test_case.range).map_err(|error| error.code); + + assert_eq!( + parsed, test_case.expected_targets, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_request_flags_when_validating_then_first_violation_is_reported() { + let test_cases = [ + RequestTestCase { + description: "model diff with a range and selector passes", + request: request(|facts| facts.from_name = Some("prod".to_owned())), + expected_code: None, + }, + RequestTestCase { + description: "two comparison modes", + request: request(|facts| { + facts.full = true; + facts.schema_only = true; + }), + expected_code: Some("C201"), + }, + RequestTestCase { + description: "duplicate keys ignoring case", + request: request(|facts| { + facts.from_name = Some("prod".to_owned()); + facts.unique_key_override = strings(&["ID", "id"]); + }), + expected_code: Some("C242"), + }, + RequestTestCase { + description: "model diff without a range", + request: request(|_| {}), + expected_code: Some("C224"), + }, + RequestTestCase { + description: "raw-query diff without a right query", + request: request(|facts| facts.left_query = true), + expected_code: Some("C218"), + }, + RequestTestCase { + description: "raw-query diff without key or unkeyed", + request: request(|facts| { + facts.left_query = true; + facts.right_query_file = true; + }), + expected_code: Some("C221"), + }, + RequestTestCase { + description: "raw-query diff rejects selectors before inputs", + request: request(|facts| { + facts.left_query = true; + facts.select = strings(&["orders"]); + }), + expected_code: Some("C213"), + }, + ]; + + for test_case in test_cases { + let code = validate_diff_request(&test_case.request) + .err() + .map(|error| error.code); + + assert_eq!( + code.as_deref(), + test_case.expected_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_blocked_models_when_building_guard_report_then_names_sizes_and_commands() { + let test_cases = [ + SizeGuardTestCase { + description: "model with cursor offers bounded command", + request: request(|facts| facts.select = strings(&["orders"])), + has_cursor: true, + expected_full: "sqb diff prod:dev --full --select orders", + expected_bounded: Some("sqb diff prod:dev --bounded --select orders"), + expected_fragments: &[ + "orders: prod 12,500,000 rows, limit 10,000,000; dev size unknown: no row count (view), limit 10,000,000", + "Run one of these instead:", + "sqb diff prod:dev --schema-only --select orders", + ], + }, + SizeGuardTestCase { + description: "model without cursor omits bounded command and quotes key flags", + request: request(|facts| { + facts.select = strings(&["tag:daily", "path:models/marts"]); + facts.unique_key_override = strings(&["order_id", "line number"]); + }), + has_cursor: false, + expected_full: "sqb diff prod:dev --full --select tag:daily path:models/marts --key order_id 'line number'", + expected_bounded: None, + expected_fragments: &["; no cursor"], + }, + SizeGuardTestCase { + description: "project dir is a global flag placed before diff", + request: request(|facts| { + facts.select = strings(&["orders"]); + facts.project_dir = Some("projects/shop".to_owned()); + }), + has_cursor: false, + expected_full: "sqb --project-dir projects/shop diff prod:dev --full --select orders", + expected_bounded: None, + expected_fragments: &[], + }, + ]; + + for test_case in test_cases { + let report = size_guard_report( + &test_case.request, + ("prod", "dev"), + "full diff stopped", + &blocked_orders(test_case.has_cursor), + ); + let lines: Vec<&str> = report.message.lines().map(str::trim).collect(); + let found: Vec<&str> = test_case + .expected_fragments + .iter() + .copied() + .filter(|fragment| report.message.contains(fragment)) + .collect(); + + assert_eq!( + ( + lines.contains(&test_case.expected_full), + test_case + .expected_bounded + .is_none_or(|bounded| lines.contains(&bounded)), + report + .help + .contains("targets.prod.diff.max_full_rows and targets.dev.diff.max_full_rows"), + report.help.contains("--bounded") == test_case.expected_bounded.is_some(), + found, + ), + ( + true, + true, + true, + true, + test_case.expected_fragments.to_vec() + ), + "{}: {}", + test_case.description, + report.message + ); + } +} + +#[test] +fn given_query_columns_when_preflighting_then_problems_are_coded() { + let test_cases = [ + ColumnsTestCase { + description: "distinct columns pass", + columns: &["id", "v"], + max_columns: Some(2), + expected_outcome: Ok(()), + }, + ColumnsTestCase { + description: "no columns", + columns: &[], + max_columns: None, + expected_outcome: Err(("C237", "left query returned no columns")), + }, + ColumnsTestCase { + description: "duplicates ignore case", + columns: &["ID", "id", "b", "B"], + max_columns: None, + expected_outcome: Err(("C238", "left query returns duplicate column names: b, id")), + }, + ColumnsTestCase { + description: "too many columns", + columns: &["a", "b"], + max_columns: Some(1), + expected_outcome: Err(( + "C239", + "left query has 2 columns, exceeding --max-columns 1", + )), + }, + ]; + + for test_case in test_cases { + let outcome = + query_column_problem("left", &strings(test_case.columns), test_case.max_columns); + let outcome = outcome.map_err(|error| (error.code, error.message)); + let outcome = outcome + .as_ref() + .copied() + .map_err(|(code, message)| (code.as_str(), message.as_str())); + + assert_eq!( + outcome, test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/tests/test_types.rs b/crates/sqlbuild-cli/src/diff_report/tests/test_types.rs index 7fab5a725f..c28b5ba3d5 100644 --- a/crates/sqlbuild-cli/src/diff_report/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/diff_report/tests/test_types.rs @@ -1,6 +1,6 @@ use sqlbuild_core::json::errors::JsonEmitError; -use crate::diff_report::models::{DiffExampleOptions, DiffModelFacts}; +use crate::diff_report::models::{DiffExampleOptions, DiffModelFacts, DiffRequestFacts}; /// Two unequal values, how they render, and what the rendering reports. pub(crate) struct EvidenceTestCase { @@ -42,3 +42,35 @@ pub(crate) struct JsonTestCase { pub(crate) document: Result, pub(crate) expected_fragment: &'static str, } + +/// A positional target range and its parsed targets or error code. +pub(crate) struct NameRangeTestCase { + pub(crate) description: &'static str, + pub(crate) range: &'static str, + pub(crate) expected_targets: Result<(String, Option), String>, +} + +/// Request flags and the code of the first rule they violate. +pub(crate) struct RequestTestCase { + pub(crate) description: &'static str, + pub(crate) request: DiffRequestFacts, + pub(crate) expected_code: Option<&'static str>, +} + +/// A blocked model's request scope and the guard's replacement commands. +pub(crate) struct SizeGuardTestCase { + pub(crate) description: &'static str, + pub(crate) request: DiffRequestFacts, + pub(crate) has_cursor: bool, + pub(crate) expected_full: &'static str, + pub(crate) expected_bounded: Option<&'static str>, + pub(crate) expected_fragments: &'static [&'static str], +} + +/// Query columns, a column limit and the preflight error code. +pub(crate) struct ColumnsTestCase { + pub(crate) description: &'static str, + pub(crate) columns: &'static [&'static str], + pub(crate) max_columns: Option, + pub(crate) expected_outcome: Result<(), (&'static str, &'static str)>, +} diff --git a/crates/sqlbuild-cli/src/diff_report/types.rs b/crates/sqlbuild-cli/src/diff_report/types.rs index c39f179e83..9cdef6303f 100644 --- a/crates/sqlbuild-cli/src/diff_report/types.rs +++ b/crates/sqlbuild-cli/src/diff_report/types.rs @@ -7,3 +7,13 @@ pub type DiffNamedValue = (String, DiffValue); /// A schema column whose type differs: name, `from` type and `to` type. pub type DiffTypeChange = (String, String, String); + +/// The phase of a raw-query diff whose user error is being classified. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum QueryDiffPhase { + Preparation, + Execution, +} + +/// Seconds spent in each named phase of a raw-query diff. +pub type PhaseSeconds = Vec<(String, f64)>; diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index c639afd309..6e185b6b5b 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -438,9 +438,9 @@ const DIFF_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const DIFF_ROUTE: Route = routed( "run_diff", - ("sqlbuild.cli.commands.main.inspection._diff", "run_diff"), + ("sqlbuild._native", "run_diff_command"), &DIFF_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const QUERY_ARGUMENTS: [Field; 6] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 7c99056e02..032882f74a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -1,8 +1,10 @@ //! `sqb` command dispatch: the native route table applied to a parsed namespace. +use crate::bindings::_helpers::diff_report::support::cli_error; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_cli::diff_report::main::parse_diff_name_range::parse_diff_name_range; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; use sqlbuild_cli::dispatch::main::unavailable_handler::unavailable_handler; use sqlbuild_cli::dispatch::models::{ @@ -17,7 +19,6 @@ const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; const WAREHOUSES_MODULE: &str = "sqlbuild._native"; const WAREHOUSE_SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const LAUNCH_RECORD_MODULE: &str = "sqlbuild.cli.entry.main._withhold_launch_record"; -const DIFF_VALIDATION_MODULE: &str = "sqlbuild.cli.commands._helpers.diff.validation"; const FORMAT_PATHS_MODULE: &str = "sqlbuild.cli.commands._helpers.lint.positional_paths"; const PATHLIB_MODULE: &str = "pathlib"; const COMMENT_PREFIX: &str = "#"; @@ -121,13 +122,13 @@ impl<'py> Invocation<'py> { if range.is_none() { return Ok(range); } - let parsed: Bound<'py, PyAny> = - imported(self.py(), DIFF_VALIDATION_MODULE, "parse_diff_name_range")? - .call1((range,))?; - parsed.get_item(match side { - DiffSide::From => 0, - DiffSide::To => 1, - }) + let py: Python<'py> = self.py(); + let (from_name, to_name) = parse_diff_name_range(&range.extract::()?) + .map_err(|error| cli_error(py, "CliUserError", &error))?; + match side { + DiffSide::From => from_name.into_bound_py_any(py), + DiffSide::To => to_name.into_bound_py_any(py), + } } fn fields(&self, fields: &[Field]) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index fe3b51108a..be33723125 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -29,7 +29,7 @@ const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_na const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main.session_warehouse_support"; const DISCOVER_CONFIGURATION_MODULE: &str = - "sqlbuild.compiler.discovery.main.discover_configuration"; + "sqlbuild.compiler.discovery.main._discover_configuration"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild.compiler.compile.main.expand_template_data"; const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs new file mode 100644 index 0000000000..164f2b106b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -0,0 +1,524 @@ +//! `sqb diff`: validate, discover, run a model or raw-query comparison and publish its output. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyTuple}; +use sqlbuild_cli::diff_report::errors::DiffUserError; +use sqlbuild_cli::diff_report::main::diff_exit_code::diff_exit_code; +use sqlbuild_cli::diff_report::main::effective_max_examples::effective_max_examples; +use sqlbuild_cli::diff_report::main::is_incomplete_query_failure::is_incomplete_query_failure; +use sqlbuild_cli::diff_report::main::is_query_diff_request::is_query_diff_request; +use sqlbuild_cli::diff_report::main::size_guard_report::size_guard_report; +use sqlbuild_cli::diff_report::main::validate_diff_request::validate_diff_request; +use sqlbuild_cli::diff_report::models::{ + DiffModelFacts, DiffRequestFacts, FullDiffModelFacts, FullDiffSideFacts, SizeGuardReport, +}; +use sqlbuild_cli::diff_report::types::QueryDiffPhase; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::diff_report::facts::diff_facts; +use crate::bindings::_helpers::diff_report::query::{ + execute_query_diff, execution_options, prepare_query_diff, project_adapter, +}; +use crate::bindings::_helpers::diff_report::registration::publish_diff_output; +use crate::bindings::_helpers::diff_report::support::{ + EXCEPTIONS_MODULE, call, caused, cli_error, error_text, is_exception, matches_class, none, + object, request_facts, user_error_parts, +}; +use crate::bindings::_helpers::frontier::encoder::emit_error; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const CONFIGURATION_MODULE: &str = "sqlbuild.compiler.discovery.main._discover_configuration"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const CONTRACT_CONSTANTS_MODULE: &str = "sqlbuild.spec.contracts.constants"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXECUTE_MODULE: &str = "sqlbuild.executor.diff.main.execute"; +const DIFF_MODELS_MODULE: &str = "sqlbuild.executor.diff.models"; +const ADAPTER_ERRORS: (&str, &str) = ("sqlbuild.adapter.contract.exceptions", "AdapterUserError"); +const EXECUTOR_INPUT_ERRORS: (&str, &str) = + ("sqlbuild.errors.contracts.exceptions", "ExecutorInputError"); +const SIZE_GUARD_ERRORS: (&str, &str) = ( + "sqlbuild.executor.diff.exceptions", + "FullDiffSizeGuardError", +); +const OUTCOME_ERRORS: (&str, &str) = (EXCEPTIONS_MODULE, "QueryDiffOutcomeError"); +const USER_ERRORS: (&str, &str) = (EXCEPTIONS_MODULE, "CliUserError"); +const ROW_DIFF_KEY_ERROR_PREFIX: &str = "row diff "; +const ROW_DIFF_KEY_ERROR_FRAGMENT: &str = " unique_key values"; + +fn user(message: &str, code: &str, help: Option<&str>) -> DiffUserError { + DiffUserError { + message: message.to_owned(), + code: code.to_owned(), + help: help.map(str::to_owned), + } +} + +/// `CliUserError` from a pure rejection. +fn rejected(py: Python<'_>, error: &DiffUserError) -> PyErr { + cli_error(py, "CliUserError", error) +} + +/// The project directory and discovered inputs, after the request's flags validate. +fn resolve_invocation<'py>( + request: &Bound<'py, PyAny>, + facts: &DiffRequestFacts, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = request.py(); + let query_mode: bool = is_query_diff_request(facts); + if let Err(error) = validate_diff_request(facts) { + return Err(cli_error( + py, + if query_mode { + "QueryDiffIncompleteError" + } else { + "CliUserError" + }, + &error, + )); + } + let project_dir: Bound<'py, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => imported(py, "pathlib", "Path")?.call_method0("cwd")?, + }; + let discovered: Bound<'py, PyAny> = if query_mode { + call( + &imported(py, CONFIGURATION_MODULE, "discover_project_configuration")?, + &[("project_dir", &project_dir)], + )? + } else { + call( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", &project_dir)], + )? + }; + call( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", &request.getattr("select")?), + ("exclude", &request.getattr("exclude")?), + ("discovered_inputs", &discovered), + ], + )?; + if facts.select.is_empty() && !query_mode { + return Err(rejected( + py, + &user("diff requires --select in v1", "C204", None), + )); + } + Ok((project_dir, discovered)) +} + +/// `message`, `code` and `help` re-raised as an incomplete or failed query-diff outcome. +fn query_outcome_error( + py: Python<'_>, + incomplete: bool, + parts: &DiffUserError, + cause: PyErr, +) -> PyErr { + let class: &str = if incomplete { + "QueryDiffIncompleteError" + } else { + "QueryDiffExecutionError" + }; + caused(py, cli_error(py, class, parts), cause) +} + +fn execution_failure(py: Python<'_>, code: &str, cause: PyErr) -> PyResult { + let parts: DiffUserError = user(&error_text(py, &cause)?, code, None); + Ok(query_outcome_error(py, false, &parts, cause)) +} + +/// Map a raw-query execution failure to its incomplete or execution-failure outcome. +fn query_execution_error(py: Python<'_>, error: PyErr) -> PyResult { + if matches_class(py, &error, OUTCOME_ERRORS)? || !is_exception(py, &error) { + return Ok(error); + } + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + if matches_class(py, &error, EXECUTOR_INPUT_ERRORS)? { + let code: String = match value.getattr_opt("code")? { + Some(code) => code.str()?.extract()?, + None => "C254".to_owned(), + }; + let help: Option = match value.getattr_opt("help")? { + Some(help) if !help.is_none() => Some(help.extract()?), + _ => None, + }; + let parts: DiffUserError = DiffUserError { + message: error_text(py, &error)?, + code, + help, + }; + return Ok(query_outcome_error(py, true, &parts, error)); + } + if matches_class(py, &error, USER_ERRORS)? { + let parts: DiffUserError = user_error_parts(&value)?; + let incomplete: bool = is_incomplete_query_failure(QueryDiffPhase::Execution, &parts.code); + return Ok(query_outcome_error(py, incomplete, &parts, error)); + } + if matches_class(py, &error, ADAPTER_ERRORS)? { + let parts: DiffUserError = user_error_parts(&value)?; + let incomplete: bool = parts.message.starts_with(ROW_DIFF_KEY_ERROR_PREFIX) + && parts.message.contains(ROW_DIFF_KEY_ERROR_FRAGMENT); + return Ok(query_outcome_error(py, incomplete, &parts, error)); + } + execution_failure(py, "C255", error) +} + +fn run_query_diff( + request: &Bound<'_, PyAny>, + invocation: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = request.py(); + let preparation: Bound<'_, PyAny> = match prepare_query_diff(request, invocation) { + Ok(preparation) => preparation, + Err(error) if matches_class(py, &error, USER_ERRORS)? => { + let parts: DiffUserError = user_error_parts(&error.value(py).clone().into_any())?; + let incomplete: bool = + is_incomplete_query_failure(QueryDiffPhase::Preparation, &parts.code); + return Err(query_outcome_error(py, incomplete, &parts, error)); + } + Err(error) if is_exception(py, &error) => { + return Err(execution_failure(py, "C253", error)?); + } + Err(error) => return Err(error), + }; + let (result, phases) = match execute_query_diff(request, &preparation) { + Ok(outcome) => outcome, + Err(error) => return Err(query_execution_error(py, error)?), + }; + let models: Vec = diff_facts(&result)?; + let schema_only: bool = request.getattr("schema_only")?.is_truthy()?; + let (outcome, exit_code) = diff_exit_code(&models, !schema_only); + let published: PyResult<()> = publish_diff_output( + request, + ( + &preparation.getattr("left_label")?.extract::()?, + &preparation.getattr("right_label")?.extract::()?, + ), + ( + preparation + .getattr("effective_max_column_examples")? + .extract()?, + preparation + .getattr("effective_max_row_only_examples")? + .extract()?, + ), + (&models, Some(phases), Some(outcome)), + ); + match published { + Ok(()) => Ok(exit_code), + Err(error) if is_exception(py, &error) => { + let parts: DiffUserError = user( + &format!( + "failed to publish query diff output: {}", + error_text(py, &error)? + ), + "C257", + None, + ); + Err(query_outcome_error(py, false, &parts, error)) + } + Err(error) => Err(error), + } +} + +fn target_max_full_rows<'py>( + discovered: &Bound<'py, PyAny>, + target: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let configured: Bound<'py, PyAny> = call( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ("project_config", &discovered.getattr("project_config")?), + ("local_config", &discovered.getattr("local_config")?), + ("target_name", target), + ], + )? + .getattr("diff")? + .getattr("max_full_rows")?; + if configured.is_none() { + return imported(py, CONTRACT_CONSTANTS_MODULE, "DEFAULT_DIFF_MAX_FULL_ROWS"); + } + if configured.is_instance_of::() && !configured.is_instance_of::() { + return Ok(configured); + } + Ok(none(py)) +} + +/// The active target when only FROM was given, rejecting a missing or identical target. +fn diff_to_target<'py>( + request: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, + from_target: &str, +) -> PyResult> { + let py: Python<'py> = request.py(); + let to_name: Bound<'py, PyAny> = request.getattr("to_name")?; + if !to_name.is_none() { + return Ok(to_name); + } + let active: Bound<'py, PyAny> = call( + &imported(py, TARGET_NAME_MODULE, "resolve_target_name")?, + &[ + ("project_config", &discovered.getattr("project_config")?), + ("local_config", &discovered.getattr("local_config")?), + ("selected_target", &none(py)), + ], + )?; + if active.is_none() { + return Err(rejected( + py, + &user( + "model diff requires FROM:TO when the project has no active target", + "C271", + Some("set default_target, or pass both targets as FROM:TO"), + ), + )); + } + if active.extract::()? == from_target { + return Err(rejected( + py, + &user( + &format!( + "diff FROM '{from_target}' is the active target, so there is nothing to compare" + ), + "C272", + Some("pass both targets as FROM:TO"), + ), + )); + } + Ok(active) +} + +fn side_facts(side: &Bound<'_, PyAny>) -> PyResult { + let detail: Bound<'_, PyAny> = side.getattr("detail")?; + Ok(FullDiffSideFacts { + target: side.getattr("target")?.extract()?, + relation: side.getattr("relation")?.extract()?, + row_count: side.getattr("row_count")?.extract()?, + max_rows: side.getattr("max_rows")?.extract()?, + exceeds_limit: side.getattr("exceeds_limit")?.is_truthy()?, + detail: if detail.is_none() { + None + } else { + Some(detail.str()?.extract()?) + }, + }) +} + +/// `DiffSizeGuardError` (C270) for the guard's blocked models. +fn size_guard_error( + py: Python<'_>, + facts: &DiffRequestFacts, + targets: (&str, &str), + error: PyErr, +) -> PyResult { + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + let mut blocked: Vec = Vec::new(); + for model in value.getattr("blocked")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + blocked.push(FullDiffModelFacts { + name: model.getattr("name")?.extract()?, + has_cursor: model.getattr("has_cursor")?.is_truthy()?, + left: side_facts(&model.getattr("left")?)?, + right: side_facts(&model.getattr("right")?)?, + }); + } + let message: String = value.getattr("message")?.extract()?; + let report: SizeGuardReport = size_guard_report(facts, targets, &message, &blocked); + let details_text: String = dumps( + &report.details, + &JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + ) + .map_err(emit_error)?; + let details: Bound<'_, PyAny> = imported(py, "json", "loads")?.call1((details_text,))?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("details", details)?; + keywords.set_item("code", "C270")?; + keywords.set_item("help", report.help)?; + let replacement: Bound<'_, PyAny> = imported(py, EXCEPTIONS_MODULE, "DiffSizeGuardError")? + .call((report.message.as_str(),), Some(&keywords))?; + Ok(caused(py, PyErr::from_value(replacement), error)) +} + +fn run_model_diff( + request: &Bound<'_, PyAny>, + facts: &DiffRequestFacts, + (project_dir, discovered): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = request.py(); + let Some(from_name) = facts.from_name.as_deref() else { + return Err(rejected( + py, + &user("model diff requires FROM or FROM:TO", "C224", None), + )); + }; + let to_target: Bound<'_, PyAny> = diff_to_target(request, discovered, from_name)?; + let to_name: String = to_target.extract()?; + let targets: Bound<'_, PyAny> = discovered.getattr("project_config")?.getattr("targets")?; + if !targets.contains(from_name)? { + return Err(rejected( + py, + &user( + &format!("unknown diff FROM target '{from_name}'"), + "C205", + None, + ), + )); + } + if !targets.contains(&to_name)? { + return Err(rejected( + py, + &user(&format!("unknown diff TO target '{to_name}'"), "C206", None), + )); + } + let from_target: Bound<'_, PyAny> = object(py, from_name)?; + let (_, adapter) = project_adapter(discovered, project_dir)?; + let cli_vars: Bound<'_, PyAny> = request.getattr("cli_vars")?; + let connection_config: Bound<'_, PyAny> = call( + &imported(py, NATIVE_MODULE, "resolve_target_connection_config")?, + &[ + ("discovered_inputs", discovered), + ("project_dir", project_dir), + ("target_name", &to_target), + ("cli_vars", &cli_vars), + ], + )?; + let resolver: Bound<'_, PyAny> = call( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", project_dir), + ("discovered_inputs", discovered), + ], + )?; + let pipeline: Bound<'_, PyTuple> = call( + &imported(py, NATIVE_MODULE, "run_diff_pipeline")?, + &[ + ("discovered_inputs", discovered), + ("adapter", &adapter), + ("from_target", &from_target), + ("to_target", &to_target), + ("resolved_connection", &connection_config), + ("no_sql_validation", &request.getattr("no_sql_validation")?), + ("select", &request.getattr("select")?), + ("exclude", &request.getattr("exclude")?), + ("cli_vars", &cli_vars), + ("external_sql_reference_resolver", &resolver), + ], + )? + .downcast_into()?; + let selected: Bound<'_, PyAny> = pipeline.get_item(2)?; + if !selected.is_truthy()? { + return Err(rejected( + py, + &user( + "no diffable models found in the selected scope", + "C207", + None, + ), + )); + } + let verbose: bool = request.getattr("verbose")?.is_truthy()?; + let column_examples: i64 = effective_max_examples(facts.max_column_examples, verbose); + let row_only_examples: i64 = effective_max_examples(facts.max_row_only_examples, verbose); + let limits: Bound<'_, PyAny> = if facts.full || facts.schema_only || facts.bounded.is_some() { + none(py) + } else { + call( + &imported(py, DIFF_MODELS_MODULE, "FullDiffSizeLimits")?, + &[ + ("left_target", &from_target), + ("right_target", &to_target), + ( + "left_max_rows", + &target_max_full_rows(discovered, &from_target)?, + ), + ( + "right_max_rows", + &target_max_full_rows(discovered, &to_target)?, + ), + ], + )? + }; + let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (&connection_config,))?; + let extras: [(&str, Bound<'_, PyAny>); 5] = [ + ("bounded", request.getattr("bounded")?), + ("max_column_examples", object(py, column_examples)?), + ("max_row_only_examples", object(py, row_only_examples)?), + ("max_models", request.getattr("max_models")?), + ("full_size_limits", limits), + ]; + let extra: Vec<(&str, &Bound<'_, PyAny>)> = + extras.iter().map(|(key, value)| (*key, value)).collect(); + let executed: PyResult> = + execution_options(request, &extra).and_then(|options| { + call( + &imported(py, EXECUTE_MODULE, "execute_diff")?, + &[ + ("adapter", &adapter), + ("connection", &connection), + ("left_project", &pipeline.get_item(0)?), + ("right_project", &pipeline.get_item(1)?), + ("selected_names", &selected), + ("options", &options), + ], + ) + }); + let executed: PyResult> = match executed { + Err(error) if matches_class(py, &error, SIZE_GUARD_ERRORS)? => { + Err(size_guard_error(py, facts, (from_name, &to_name), error)?) + } + other => other, + }; + adapter.call_method1("close", (&connection,))?; + let result: Bound<'_, PyAny> = executed?; + let models: Vec = diff_facts(&result)?; + publish_diff_output( + request, + (from_name, &to_name), + (column_examples, row_only_examples), + (&models, None, None), + )?; + Ok(diff_exit_code(&models, false).1) +} + +/// Run one `sqb diff` request; the exit code. +#[pyfunction] +#[pyo3(signature = (*, request))] +pub(crate) fn run_diff_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let facts: DiffRequestFacts = request_facts(request)?; + let query_mode: bool = is_query_diff_request(&facts); + let (project_dir, discovered) = match resolve_invocation(request, &facts) { + Ok(invocation) => invocation, + Err(error) + if query_mode + && !matches_class(py, &error, OUTCOME_ERRORS)? + && is_exception(py, &error) => + { + return Err(execution_failure(py, "C252", error)?); + } + Err(error) => return Err(error), + }; + if query_mode { + return run_query_diff(request, (&project_dir, &discovered)); + } + run_model_diff(request, &facts, (&project_dir, &discovered)) +} + +/// Materialize, compare and clean one prepared raw-query pair; the diff result. +#[pyfunction] +#[pyo3(name = "execute_query_diff", signature = (*, request, preparation))] +pub(crate) fn execute_prepared_query_diff<'py>( + request: &Bound<'py, PyAny>, + preparation: &Bound<'py, PyAny>, +) -> PyResult> { + execute_query_diff(request, preparation).map(|(result, _)| result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs index 35d9ab8112..a2dea05e2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs @@ -1,2 +1,5 @@ +pub(crate) mod command; pub(crate) mod facts; +pub(crate) mod query; pub(crate) mod registration; +pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs new file mode 100644 index 0000000000..6efc6275fc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -0,0 +1,772 @@ +//! Raw-query `sqb diff`: preparation, artifact materialization, comparison and cleanup. + +use std::time::Instant; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_cli::diff_report::errors::DiffUserError; +use sqlbuild_cli::diff_report::main::effective_max_examples::effective_max_examples; +use sqlbuild_cli::diff_report::main::query_column_problem::query_column_problem; +use sqlbuild_cli::diff_report::main::read_only_query_problem::read_only_query_problem; +use sqlbuild_cli::diff_report::models::QueryShapeFacts; +use sqlbuild_cli::diff_report::types::PhaseSeconds; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::diff_report::support::{ + call, cli_error, error_text, none, object, progress_line, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const NAMESPACE_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target_namespace"; +const RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; +const POLYGLOT_MODULE: &str = "sqlbuild.compiler.sql_analysis.main.import_polyglot_sql"; +const COMMAND_MODELS_MODULE: &str = "sqlbuild.cli.commands.models"; +const LIFECYCLE_MODULE: &str = "sqlbuild.executor.diff.classes.query_artifact_lifecycle"; +const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; +const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +const EXECUTE_QUERY_MODULE: &str = "sqlbuild.executor.diff.main._execute_query"; +const DIFF_MODELS_MODULE: &str = "sqlbuild.executor.diff.models"; +const DIFF_CONFIG_MODULE: &str = "sqlbuild.executor.diff.main._config"; +const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; + +fn failure(py: Python<'_>, message: String, code: &str, help: Option<&str>) -> PyErr { + cli_error( + py, + "CliUserError", + &DiffUserError { + message, + code: code.to_owned(), + help: help.map(str::to_owned), + }, + ) +} + +fn user_failure(py: Python<'_>, error: DiffUserError) -> PyErr { + cli_error(py, "CliUserError", &error) +} + +/// The adapter of the effective adapter name, resolved for the project directory. +pub(crate) fn project_adapter<'py>( + discovered: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult<(String, Bound<'py, PyAny>)> { + let py: Python<'py> = discovered.py(); + let name: Bound<'py, PyAny> = call( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", &discovered.getattr("project_config")?), + ("local_config", &discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = call( + &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, + &[("adapter_name", &name), ("project_dir", project_dir)], + )?; + Ok((name.extract()?, adapter)) +} + +fn query_input( + py: Python<'_>, + inline: Bound<'_, PyAny>, + file: Bound<'_, PyAny>, + label: &str, +) -> PyResult { + if !inline.is_none() { + return inline.extract(); + } + if file.is_none() { + return Err(failure( + py, + format!("raw-query diff requires a {label} query"), + "C229", + None, + )); + } + if !file.call_method0("exists")?.is_truthy()? || !file.call_method0("is_file")?.is_truthy()? { + return Err(failure( + py, + format!("{label} query file does not exist: {}", file.str()?), + "C230", + None, + )); + } + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("encoding", "utf-8")?; + match file.call_method("read_text", (), Some(&keywords)) { + Ok(text) => text.extract(), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + let replacement: PyErr = failure( + py, + format!( + "{label} query file could not be read as UTF-8: {}", + file.str()? + ), + "C231", + None, + ); + replacement.set_cause(py, Some(error)); + Err(replacement) + } + Err(error) => Err(error), + } +} + +fn node_key(node: &Bound<'_, PyAny>) -> PyResult { + match node.getattr_opt("key")? { + Some(key) => Ok(key.str()?.to_string().to_lowercase()), + None => Ok(String::new()), + } +} + +fn read_only_query( + py: Python<'_>, + sql: &str, + label: &str, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let mut normalized: &str = sql.trim(); + if let Some(stripped) = normalized.strip_suffix(';') { + normalized = stripped.trim_end(); + } + if normalized.is_empty() { + return Err(failure( + py, + format!("{label} query must not be empty"), + "C232", + None, + )); + } + let polyglot: Bound<'_, PyAny> = + imported(py, POLYGLOT_MODULE, "import_polyglot_sql")?.call0()?; + if polyglot.is_none() { + return Err(failure( + py, + "raw-query diff requires the bundled SQL parser".to_owned(), + "C233", + None, + )); + } + let dialect: Bound<'_, PyAny> = adapter.call_method0("sql_analysis_dialect")?; + let dialect: Bound<'_, PyAny> = if dialect.is_truthy()? { + dialect + } else { + object(py, "generic")? + }; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("dialect", dialect)?; + let expression: Bound<'_, PyAny> = + match polyglot.call_method("parse_one", (normalized,), Some(&keywords)) { + Ok(expression) => expression, + Err(error) if error.is_instance_of::(py) => { + let replacement: PyErr = failure( + py, + format!( + "{label} query is not one valid SQL statement: {}", + error_text(py, &error)? + ), + "C234", + None, + ); + replacement.set_cause(py, Some(error)); + return Err(replacement); + } + Err(error) => return Err(error), + }; + let mut node_keys: Vec = Vec::new(); + let root_key: String = node_key(&expression)?; + let read_only_root: bool = + ["except", "intersect", "select", "union", "values"].contains(&root_key.as_str()); + if read_only_root { + for node in expression.call_method0("walk")?.try_iter()? { + node_keys.push(node_key(&node?)?); + } + } + let arguments: Bound<'_, PyAny> = expression.getattr_opt("args")?.unwrap_or_else(|| none(py)); + let writes_or_locks: bool = match arguments.downcast::() { + Ok(arguments) => { + arguments + .get_item("into")? + .is_some_and(|into| !into.is_none()) + || arguments + .get_item("locks")? + .map_or(Ok(false), |locks| locks.is_truthy())? + } + Err(_) => false, + }; + read_only_query_problem( + label, + &QueryShapeFacts { + root_key, + node_keys, + writes_or_locks, + }, + ) + .map_err(|error| user_failure(py, error))?; + Ok(normalized.to_owned()) +} + +/// Resolve raw SQL, the active target namespace, the adapter and its connection. +pub(crate) fn prepare_query_diff<'py>( + request: &Bound<'py, PyAny>, + (project_dir, discovered): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let (adapter_name, adapter) = project_adapter(discovered, project_dir)?; + let selected_target: Bound<'py, PyAny> = request.getattr("selected_target")?; + let cli_vars: Bound<'py, PyAny> = request.getattr("cli_vars")?; + let config: Bound<'py, PyAny> = call( + &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, + &[ + ("discovered_inputs", discovered), + ("project_dir", project_dir), + ("selected_target", &selected_target), + ("cli_vars", &cli_vars), + ], + )?; + let database: Bound<'py, PyAny> = if NAMESPACELESS_ADAPTERS.contains(&adapter_name.as_str()) { + none(py) + } else { + config.call_method1("get", ("database",))? + }; + let namespace: Bound<'py, PyAny> = call( + &imported(py, NAMESPACE_MODULE, "build_effective_target_namespace")?, + &[ + ("discovered_inputs", discovered), + ("selected_target", &selected_target), + ("connection_database", &database), + ( + "connection_schema", + &config.call_method1("get", ("schema",))?, + ), + ( + "default_database", + &adapter.call_method0("default_database")?, + ), + ("default_schema", &adapter.call_method0("default_schema")?), + ], + )?; + let (target_name, _target, database, schema): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = namespace.extract()?; + if schema.is_none() { + return Err(failure( + py, + "raw-query diff requires an active target schema".to_owned(), + "C227", + Some("configure the selected target schema before running the comparison"), + )); + } + let left_sql: String = query_input( + py, + request.getattr("left_query")?, + request.getattr("left_query_file")?, + "left", + )?; + let right_sql: String = query_input( + py, + request.getattr("right_query")?, + request.getattr("right_query_file")?, + "right", + )?; + let left_sql: String = read_only_query(py, &left_sql, "left", &adapter)?; + let right_sql: String = read_only_query(py, &right_sql, "right", &adapter)?; + let runtime: Bound<'py, PyTuple> = call( + &imported(py, RUNTIME_MODULE, "build_effective_runtime_config")?, + &[ + ("discovered_inputs", discovered), + ("selected_target", &selected_target), + ("cli_vars", &cli_vars), + ], + )? + .downcast_into()?; + let label = |name: &str, default: &str| -> PyResult { + let value: Bound<'py, PyAny> = request.getattr(name)?; + let text: String = if value.is_truthy()? { + value.extract()? + } else { + default.to_owned() + }; + Ok(text.trim().to_owned()) + }; + let verbose: bool = request.getattr("verbose")?.is_truthy()?; + let examples = |name: &str| -> PyResult> { + object( + py, + effective_max_examples(request.getattr(name)?.extract()?, verbose), + ) + }; + call( + &imported(py, COMMAND_MODELS_MODULE, "QueryDiffPreparation")?, + &[ + ("adapter", &adapter), + ("connection_config", &config), + ("left_sql", &object(py, left_sql)?), + ("right_sql", &object(py, right_sql)?), + ( + "left_label", + &object(py, label("left_label", "left query")?)?, + ), + ( + "right_label", + &object(py, label("right_label", "right query")?)?, + ), + ("selected_target", &target_name), + ("database", &database), + ("schema", &schema), + ("run_id", &runtime.get_item(2)?), + ( + "artifact_ttl", + &discovered + .getattr("project_config")? + .getattr("diff")? + .getattr("query_artifact_ttl")?, + ), + ( + "effective_max_column_examples", + &examples("max_column_examples")?, + ), + ( + "effective_max_row_only_examples", + &examples("max_row_only_examples")?, + ), + ], + ) +} + +fn seconds(started: Instant) -> f64 { + started.elapsed().as_secs_f64() +} + +/// `RowDiffSamplingOverride` and the comparison options a request asks for. +pub(crate) fn execution_options<'py>( + request: &Bound<'py, PyAny>, + extra: &[(&str, &Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = request.py(); + let attribute = |name: &str| request.getattr(name); + let sampling: Bound<'py, PyAny> = call( + &imported(py, DIFF_MODELS_MODULE, "RowDiffSamplingOverride")?, + &[ + ("row_limit", &attribute("sample_rows")?), + ("seed", &attribute("sample_seed")?), + ("exhaustive", &attribute("exhaustive")?), + ], + )?; + let tolerances: Bound<'py, PyAny> = call( + &imported(py, DIFF_CONFIG_MODULE, "parse_cli_tolerance_overrides")?, + &[("values", &attribute("tolerance_overrides")?)], + )?; + let schema_only: Bound<'py, PyAny> = attribute("schema_only")?; + let collect: Bound<'py, PyAny> = object(py, !schema_only.is_truthy()?)?; + let mut pairs: Vec<(&str, &Bound<'py, PyAny>)> = + vec![("schema_only", &schema_only), ("collect_samples", &collect)]; + let columns: Bound<'py, PyAny> = attribute("max_columns")?; + let key: Bound<'py, PyAny> = attribute("unique_key_override")?; + let unkeyed: Bound<'py, PyAny> = attribute("unkeyed")?; + let excluded: Bound<'py, PyAny> = attribute("excluded_columns_override")?; + pairs.extend([ + ("max_columns", &columns), + ("sampling_override", &sampling), + ("unique_key_override", &key), + ("unkeyed", &unkeyed), + ("excluded_columns_override", &excluded), + ("tolerance_overrides", &tolerances), + ]); + pairs.extend_from_slice(extra); + call( + &imported(py, DIFF_MODELS_MODULE, "DiffExecutionOptions")?, + &pairs, + ) +} + +/// Materialize, compare and clean one raw-query pair; the result and per-phase seconds. +pub(crate) fn execute_query_diff<'py>( + request: &Bound<'py, PyAny>, + preparation: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, PhaseSeconds)> { + let py: Python<'py> = request.py(); + let adapter: Bound<'py, PyAny> = preparation.getattr("adapter")?; + let total_started: Instant = Instant::now(); + let mut phases: Vec<(String, f64)> = Vec::new(); + progress_line(py, "Query diff connection START")?; + let connection_started: Instant = Instant::now(); + let connection: Bound<'py, PyAny> = + match adapter.call_method1("connect", (preparation.getattr("connection_config")?,)) { + Ok(connection) => connection, + Err(error) => { + progress_line( + py, + &format!( + "Query diff connection ERROR ({:.2}s)", + seconds(connection_started) + ), + )?; + progress_line( + py, + &format!("Query diff total ERROR ({:.2}s)", seconds(total_started)), + )?; + return Err(error); + } + }; + phases.push(("connection".to_owned(), seconds(connection_started))); + progress_line( + py, + &format!("Query diff connection OK ({:.2}s)", phases[0].1), + )?; + let lifecycle: Bound<'py, PyAny> = + imported(py, LIFECYCLE_MODULE, "QueryDiffArtifactLifecycle")?; + let artifact = |side: &str| -> PyResult> { + call( + &lifecycle.getattr("build")?, + &[ + ("adapter", &adapter), + ("run_id", &preparation.getattr("run_id")?), + ("side", &object(py, side)?), + ("database", &preparation.getattr("database")?), + ("schema", &preparation.getattr("schema")?), + ], + ) + }; + let left: Bound<'py, PyAny> = artifact("left")?; + let right: Bound<'py, PyAny> = artifact("right")?; + let mut created: Vec> = Vec::new(); + let outcome: PyResult> = compare_artifacts( + (request, preparation), + (&connection, &lifecycle), + (&left, &right), + (&mut created, &mut phases), + ); + let cleanup_started: Instant = Instant::now(); + progress_line(py, "Query diff immediate cleanup START")?; + let mut cleanup_errors: Vec = Vec::new(); + for item in created.iter().rev() { + if let Err(error) = call( + &lifecycle.getattr("cleanup")?, + &[ + ("adapter", &adapter), + ("connection", &connection), + ("artifact", item), + ], + ) { + cleanup_errors.push(error); + } + } + if let Err(error) = adapter.call_method1("close", (&connection,)) { + cleanup_errors.push(error); + } + let cleanup_seconds: f64 = seconds(cleanup_started); + if cleanup_errors.is_empty() { + progress_line( + py, + &format!( + "Query diff immediate cleanup OK ({} removed; {cleanup_seconds:.2}s)", + created.len() + ), + )?; + } else { + progress_line( + py, + &format!( + "Query diff immediate cleanup ERROR ({} failed; {cleanup_seconds:.2}s)", + cleanup_errors.len() + ), + )?; + } + phases.push(("cleanup".to_owned(), cleanup_seconds)); + let total: f64 = seconds(total_started); + phases.push(("total".to_owned(), total)); + let status: &str = if outcome.is_err() || !cleanup_errors.is_empty() { + "ERROR" + } else { + "OK" + }; + progress_line(py, &format!("Query diff total {status} ({total:.2}s)"))?; + if !cleanup_errors.is_empty() { + return Err(cleanup_errors.remove(0)); + } + let model: Bound<'py, PyAny> = outcome?; + if model.is_none() { + return Err(cli_error( + py, + "QueryDiffExecutionError", + &DiffUserError { + message: "query diff comparison completed without a result".to_owned(), + code: "C256".to_owned(), + help: None, + }, + )); + } + let result: Bound<'py, PyAny> = call( + &imported(py, DIFF_MODELS_MODULE, "DiffExecutionResult")?, + &[("model_results", &PyTuple::new(py, [model])?.into_any())], + )?; + Ok((result, phases)) +} + +/// Reconcile, inspect, materialize both sides and compare them; the model result. +fn compare_artifacts<'py>( + (request, preparation): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (connection, lifecycle): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (left, right): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (created, phases): (&mut Vec>, &mut Vec<(String, f64)>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let adapter: Bound<'py, PyAny> = preparation.getattr("adapter")?; + phases.push(( + "reconciliation".to_owned(), + reconcile(preparation, connection, lifecycle)?, + )); + let (expires_at, inspection) = inspect(request, preparation, connection, (left, right))?; + phases.push(("inspection".to_owned(), inspection)); + progress_line(py, "Query diff materialization START")?; + let started: Instant = Instant::now(); + for (artifact, sql) in [(left, "left_sql"), (right, "right_sql")] { + if let Err(error) = call( + &lifecycle.getattr("materialize")?, + &[ + ("adapter", &adapter), + ("connection", connection), + ("artifact", artifact), + ("sql", &preparation.getattr(sql)?), + ("expires_at", &expires_at), + ], + ) { + progress_line( + py, + &format!( + "Query diff materialization ERROR ({:.2}s)", + seconds(started) + ), + )?; + return Err(error); + } + created.push(artifact.clone()); + } + let materialization: f64 = seconds(started); + phases.push(("materialization".to_owned(), materialization)); + progress_line( + py, + &format!( + "Query diff materialization OK ({}; expires {}; {materialization:.2}s)", + preparation.getattr("schema")?.str()?, + expires_at.call_method0("isoformat")?.str()? + ), + )?; + progress_line(py, "Query diff comparison START")?; + let started: Instant = Instant::now(); + let name: String = format!( + "{} vs {}", + preparation.getattr("left_label")?.str()?, + preparation.getattr("right_label")?.str()? + ); + let pairs: [(&str, Bound<'py, PyAny>); 3] = [ + ( + "max_column_examples", + preparation.getattr("effective_max_column_examples")?, + ), + ( + "max_row_only_examples", + preparation.getattr("effective_max_row_only_examples")?, + ), + ("comparison_name", object(py, name)?), + ]; + let extra: Vec<(&str, &Bound<'py, PyAny>)> = + pairs.iter().map(|(key, value)| (*key, value)).collect(); + let compared: PyResult> = + execution_options(request, &extra).and_then(|options| { + call( + &imported(py, EXECUTE_QUERY_MODULE, "execute_query_diff")?, + &[ + ("adapter", &adapter), + ("connection", connection), + ("left_relation", &left.getattr("relation")?), + ("right_relation", &right.getattr("relation")?), + ("options", &options), + ], + ) + }); + let comparison: f64 = seconds(started); + phases.push(("comparison".to_owned(), comparison)); + match compared { + Ok(model) => { + progress_line( + py, + &format!("Query diff comparison OK ({comparison:.2}s)"), + )?; + Ok(model) + } + Err(error) => { + progress_line( + py, + &format!("Query diff comparison ERROR ({comparison:.2}s)"), + )?; + Err(error) + } + } +} + +fn reconcile( + preparation: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + lifecycle: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = preparation.py(); + progress_line(py, "Query diff reconciliation START")?; + let started: Instant = Instant::now(); + let now: Bound<'_, PyAny> = imported(py, "datetime", "datetime")? + .call_method1("now", (imported(py, "datetime", "UTC")?,))?; + let cleaned: PyResult> = call( + &lifecycle.getattr("cleanup_expired")?, + &[ + ("adapter", &preparation.getattr("adapter")?), + ("connection", connection), + ("database", &preparation.getattr("database")?), + ("schema", &preparation.getattr("schema")?), + ("now", &now), + ], + ); + let elapsed: f64 = seconds(started); + let result: Bound<'_, PyAny> = match cleaned { + Ok(result) => result, + Err(error) if error.is_instance_of::(py) => { + progress_line( + py, + &format!( + "Query diff reconciliation WARNING recovery cleanup could not inspect historical ownership evidence: {} ({elapsed:.2}s)", + error_text(py, &error)? + ), + )?; + return Ok(elapsed); + } + Err(error) => return Err(error), + }; + progress_line( + py, + &format!( + "Query diff reconciliation OK ({} expired; {elapsed:.2}s)", + result.getattr("cleaned")?.len()? + ), + )?; + for relation in result.getattr("untracked")?.try_iter()? { + progress_line( + py, + &format!( + "Query diff reconciliation WARNING reserved-name relation lacks matching ownership evidence: {}", + relation?.getattr("name")?.str()? + ), + )?; + } + Ok(elapsed) +} + +fn inspect<'py>( + request: &Bound<'py, PyAny>, + preparation: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + artifacts: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyAny>, f64)> { + let py: Python<'py> = request.py(); + progress_line(py, "Query diff inspection START")?; + let started: Instant = Instant::now(); + match inspect_inputs(request, preparation, connection, artifacts) { + Ok(ttl) => { + let elapsed: f64 = seconds(started); + progress_line(py, &format!("Query diff inspection OK ({elapsed:.2}s)"))?; + let now: Bound<'py, PyAny> = imported(py, "datetime", "datetime")? + .call_method1("now", (imported(py, "datetime", "UTC")?,))?; + Ok((ttl.call_method1("add_to", (now,))?, elapsed)) + } + Err(error) => { + progress_line( + py, + &format!("Query diff inspection ERROR ({:.2}s)", seconds(started)), + )?; + Err(error) + } + } +} + +/// Refuse existing artifacts, an invalid TTL and unusable query columns; the parsed TTL. +fn inspect_inputs<'py>( + request: &Bound<'py, PyAny>, + preparation: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + (left, right): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let adapter: Bound<'py, PyAny> = preparation.getattr("adapter")?; + let location = |artifact: &Bound<'py, PyAny>| -> PyResult> { + PyTuple::new( + py, + [ + artifact.getattr("database")?, + artifact.getattr("schema")?, + artifact.getattr("name")?, + ], + ) + }; + let locations: Bound<'py, PyTuple> = PyTuple::new(py, [location(left)?, location(right)?])?; + let lookup: Bound<'py, PyAny> = call( + &imported(py, RELATION_LOOKUP_MODULE, "build_relation_lookup")?, + &[ + ("adapter", &adapter), + ("connection", connection), + ("locations", &locations.into_any()), + ], + )?; + for artifact in [left, right] { + let exists: Bound<'py, PyAny> = call( + &lookup.getattr("exists")?, + &[ + ("database", &artifact.getattr("database")?), + ("schema", &artifact.getattr("schema")?), + ("name", &artifact.getattr("name")?), + ], + )?; + if exists.is_truthy()? { + return Err(failure( + py, + format!( + "query-diff artifact already exists: {}", + artifact.getattr("relation")?.str()? + ), + "C245", + Some("wait for its TTL cleanup or investigate the reserved-name relation"), + )); + } + } + let ttl: Bound<'py, PyAny> = imported(py, CURSOR_MODELS_MODULE, "Duration")? + .call_method1("parse", (preparation.getattr("artifact_ttl")?,))?; + if ttl.is_none() { + return Err(failure( + py, + "diff.query_artifact_ttl must be a positive duration".to_owned(), + "C228", + None, + )); + } + let max_columns: Option = request.getattr("max_columns")?.extract()?; + for (sql, label) in [("left_sql", "left_label"), ("right_sql", "right_label")] { + let columns: Vec = call( + &adapter.getattr("query_column_names")?, + &[ + ("connection", connection), + ("sql", &preparation.getattr(sql)?), + ], + )? + .extract()?; + let label: String = preparation.getattr(label)?.extract()?; + query_column_problem(&label, &columns, max_columns) + .map_err(|error| user_failure(py, error))?; + } + Ok(ttl) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs index 406899bf13..644245c366 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs @@ -5,7 +5,6 @@ use pyo3::prelude::{ wrap_pyfunction, }; use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_cli::diff_report::main::diff_exit_code::diff_exit_code; use sqlbuild_cli::diff_report::main::render_diff_error_json::render_diff_error_json as error_document; use sqlbuild_cli::diff_report::main::render_diff_json::render_diff_json; use sqlbuild_cli::diff_report::main::render_diff_text::render_diff_text; @@ -14,8 +13,12 @@ use sqlbuild_cli::diff_report::models::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::diff_report::command::{ + execute_prepared_query_diff, run_diff_command, +}; use crate::bindings::_helpers::diff_report::facts::{diff_facts, json_of}; use crate::bindings::_helpers::frontier::encoder::emit_error; +use sqlbuild_cli::diff_report::types::PhaseSeconds; const DEFAULT_MAX_VALUE_LENGTH: usize = 160; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; @@ -32,29 +35,8 @@ const TEXT_OPTIONS: (&[&str], &[&str]) = ( ], &["max_value_length", "suppress_example_values", "outcome"], ); -const JSON_OPTIONS: (&[&str], &[&str]) = ( - &["result", "from_label", "to_label"], - &[ - "max_value_length", - "suppress_example_values", - "phase_seconds", - "outcome", - ], -); const ERROR_OPTIONS: (&[&str], &[&str]) = (&["status", "code", "message"], &["help_text", "size_guard"]); -const WRITE_OPTIONS: (&[&str], &[&str]) = ( - &[ - "request", - "from_label", - "to_label", - "max_column_examples", - "max_row_only_examples", - "result", - ], - &["phase_seconds", "outcome"], -); -const EXIT_OPTIONS: (&[&str], &[&str]) = (&["result"], &["query_row_comparison"]); fn optional_string(value: Bound<'_, PyAny>) -> PyResult> { if value.is_none() { @@ -132,19 +114,6 @@ fn render_diff_output(py: Python<'_>, options: Option>) -> PyR )) } -/// Render one stable structured diff document. -#[pyfunction] -#[pyo3(signature = (**options))] -fn render_diff_json_output(py: Python<'_>, options: Option>) -> PyResult { - let arguments: KeywordArguments<'_> = - KeywordArguments::read(py, "render_diff_json_output", options, JSON_OPTIONS)?; - let models: Vec = diff_facts(&arguments.value("result")?)?; - json_text( - &models, - &document_options(&arguments, example_options(&arguments)?)?, - ) -} - /// Render a machine-readable incomplete or execution-failure outcome. #[pyfunction] #[pyo3(signature = (**options))] @@ -183,12 +152,13 @@ fn write_stdout(py: Python<'_>, text: &str) -> PyResult<()> { } /// Write the requested JSON file, then print the JSON document or the terminal summary. -#[pyfunction] -#[pyo3(signature = (**options))] -fn write_diff_output(py: Python<'_>, options: Option>) -> PyResult<()> { - let arguments: KeywordArguments<'_> = - KeywordArguments::read(py, "write_diff_output", options, WRITE_OPTIONS)?; - let request: Bound<'_, PyAny> = arguments.value("request")?; +pub(crate) fn publish_diff_output( + request: &Bound<'_, PyAny>, + (from_label, to_label): (&str, &str), + (max_column_examples, max_row_only_examples): (i64, i64), + (models, phase_seconds, outcome): (&[DiffModelFacts], Option, Option), +) -> PyResult<()> { + let py: Python<'_> = request.py(); let max_value_length: Bound<'_, PyAny> = request.getattr("max_value_length")?; let examples: DiffExampleOptions = DiffExampleOptions { max_value_length: if request.getattr("full_example_values")?.is_truthy()? { @@ -200,8 +170,13 @@ fn write_diff_output(py: Python<'_>, options: Option>) -> PyRe }, suppress_values: request.getattr("suppress_example_values")?.is_truthy()?, }; - let document: DiffDocumentOptions = document_options(&arguments, examples)?; - let models: Vec = diff_facts(&arguments.value("result")?)?; + let document: DiffDocumentOptions = DiffDocumentOptions { + from_label: from_label.to_owned(), + to_label: to_label.to_owned(), + examples, + outcome, + phase_seconds, + }; let path: Bound<'_, PyAny> = request.getattr("json_output_path")?; if !path.is_none() { let keywords: Bound<'_, PyDict> = PyDict::new(py); @@ -213,12 +188,12 @@ fn write_diff_output(py: Python<'_>, options: Option>) -> PyRe encoding.set_item("encoding", "utf-8")?; path.call_method( "write_text", - (format!("{}\n", json_text(&models, &document)?),), + (format!("{}\n", json_text(models, &document)?),), Some(&encoding), )?; } if request.getattr("json_output")?.is_truthy()? { - return write_stdout(py, &json_text(&models, &document)?); + return write_stdout(py, &json_text(models, &document)?); } let use_color: bool = !request.getattr("no_color")?.is_truthy()? && imported(py, COLOR_MODULE, "supports_color")? @@ -227,39 +202,22 @@ fn write_diff_output(py: Python<'_>, options: Option>) -> PyRe write_stdout( py, &render_diff_text( - &models, + models, &document, &DiffTextOptions { - mode_label: mode_label(&request)?, + mode_label: mode_label(request)?, use_color, verbose: request.getattr("verbose")?.is_truthy()?, - max_column_examples: arguments.value("max_column_examples")?.extract()?, - max_row_only_examples: arguments.value("max_row_only_examples")?.extract()?, + max_column_examples: usize::try_from(max_column_examples).unwrap_or_default(), + max_row_only_examples: usize::try_from(max_row_only_examples).unwrap_or_default(), }, ), ) } -/// `(outcome, exit_code)` of a finished diff, with query row-comparison semantics on request. -#[pyfunction] -#[pyo3(signature = (**options))] -fn resolve_diff_outcome( - py: Python<'_>, - options: Option>, -) -> PyResult<(String, i32)> { - let arguments: KeywordArguments<'_> = - KeywordArguments::read(py, "resolve_diff_outcome", options, EXIT_OPTIONS)?; - let models: Vec = diff_facts(&arguments.value("result")?)?; - Ok(diff_exit_code( - &models, - arguments.flag("query_row_comparison")?, - )) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(render_diff_output, module)?)?; - module.add_function(wrap_pyfunction!(render_diff_json_output, module)?)?; module.add_function(wrap_pyfunction!(render_diff_error_json, module)?)?; - module.add_function(wrap_pyfunction!(write_diff_output, module)?)?; - module.add_function(wrap_pyfunction!(resolve_diff_outcome, module)?) + module.add_function(wrap_pyfunction!(run_diff_command, module)?)?; + module.add_function(wrap_pyfunction!(execute_prepared_query_diff, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs new file mode 100644 index 0000000000..77f7cfe9b4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs @@ -0,0 +1,160 @@ +//! Python calls, errors and request facts shared by the `sqb diff` command phases. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_cli::diff_report::errors::DiffUserError; +use sqlbuild_cli::diff_report::models::DiffRequestFacts; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; + +/// `name(**pairs)` with keyword arguments. +pub(crate) fn call<'py>( + callable: &Bound<'py, PyAny>, + pairs: &[(&str, &Bound<'py, PyAny>)], +) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(callable.py()); + for (name, value) in pairs { + keywords.set_item(*name, *value)?; + } + callable.call((), Some(&keywords)) +} + +pub(crate) fn none(py: Python<'_>) -> Bound<'_, PyAny> { + py.None().into_bound(py) +} + +pub(crate) fn object<'py, T: pyo3::IntoPyObjectExt<'py>>( + py: Python<'py>, + value: T, +) -> PyResult> { + value.into_bound_py_any(py) +} + +/// `class(message, code=code, help=help)` from the CLI exceptions module. +pub(crate) fn cli_error(py: Python<'_>, class: &str, error: &DiffUserError) -> PyErr { + let raised = (|| -> PyResult> { + let help: Bound<'_, PyAny> = object(py, error.help.as_deref())?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", error.code.as_str())?; + keywords.set_item("help", help)?; + imported(py, EXCEPTIONS_MODULE, class)?.call((error.message.as_str(),), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// The `message`, `code` and `help` of a raised `CliUserError`. +pub(crate) fn user_error_parts(error: &Bound<'_, PyAny>) -> PyResult { + let help: Bound<'_, PyAny> = error.getattr("help")?; + Ok(DiffUserError { + message: error.getattr("message")?.extract()?, + code: error.getattr("code")?.str()?.extract()?, + help: if help.is_none() { + None + } else { + Some(help.extract()?) + }, + }) +} + +/// `error` re-raised as `class` with `error` as its cause. +pub(crate) fn caused(py: Python<'_>, replacement: PyErr, cause: PyErr) -> PyErr { + replacement.set_cause(py, Some(cause)); + replacement +} + +pub(crate) fn matches_class( + py: Python<'_>, + error: &PyErr, + (module, class): (&str, &str), +) -> PyResult { + error.value(py).is_instance(&imported(py, module, class)?) +} + +/// Whether `error` is an `Exception` rather than a `BaseException` such as an interrupt. +pub(crate) fn is_exception(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) +} + +pub(crate) fn error_text(py: Python<'_>, error: &PyErr) -> PyResult { + error.value(py).str()?.extract() +} + +/// Write one line to `sys.stderr`, as `print(..., file=sys.stderr)` does. +pub(crate) fn progress_line(py: Python<'_>, line: &str) -> PyResult<()> { + imported(py, "sys", "stderr")?.call_method1("write", (format!("{line}\n"),))?; + Ok(()) +} + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +fn texts(value: Bound<'_, PyAny>) -> PyResult> { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(item?.str()?.extract()?); + } + Ok(items) +} + +/// The facts of one `DiffCommandRequest` that validation and the size guard read. +pub(crate) fn request_facts(request: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = request.py(); + let attribute = |name: &str| request.getattr(name); + let present = |name: &str| -> PyResult { Ok(!attribute(name)?.is_none()) }; + let number = |name: &str| -> PyResult> { attribute(name)?.extract() }; + let flag = |name: &str| -> PyResult { attribute(name)?.is_truthy() }; + let variables: Bound<'_, PyAny> = attribute("cli_vars")?; + let cli_vars_json: Option = if variables.is_truthy()? { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("sort_keys", true)?; + Some( + imported(py, "json", "dumps")? + .call((variables,), Some(&keywords))? + .extract()?, + ) + } else { + None + }; + Ok(DiffRequestFacts { + from_name: optional_text(attribute("from_name")?)?, + to_name: optional_text(attribute("to_name")?)?, + full: flag("full")?, + schema_only: flag("schema_only")?, + bounded: optional_text(attribute("bounded")?)?, + max_column_examples: number("max_column_examples")?, + max_row_only_examples: number("max_row_only_examples")?, + sample_rows: number("sample_rows")?, + sample_seed: number("sample_seed")?, + exhaustive: flag("exhaustive")?, + max_models: number("max_models")?, + max_columns: number("max_columns")?, + max_value_length: number("max_value_length")?, + full_example_values: flag("full_example_values")?, + suppress_example_values: flag("suppress_example_values")?, + select: texts(attribute("select")?)?, + exclude: texts(attribute("exclude")?)?, + selected_target: optional_text(attribute("selected_target")?)?, + left_query: present("left_query")?, + left_query_file: present("left_query_file")?, + right_query: present("right_query")?, + right_query_file: present("right_query_file")?, + left_label: optional_text(attribute("left_label")?)?, + right_label: optional_text(attribute("right_label")?)?, + unique_key_override: texts(attribute("unique_key_override")?)?, + unkeyed: flag("unkeyed")?, + excluded_columns_override: texts(attribute("excluded_columns_override")?)?, + tolerance_overrides: texts(attribute("tolerance_overrides")?)?, + cli_vars_json, + project_dir: optional_text(attribute("project_dir")?)?, + }) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5505a95cf1..85d17eeee3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -27,6 +27,7 @@ from sqlbuild.cli.commands.models import ( LineageCommandRequest, LoadCommandRequest, PlaygroundCommandRequest, + QueryDiffPreparation, RefactorCommandRequest, ScopeCommandRequest, TestCommandRequest, @@ -1538,16 +1539,6 @@ def render_diff_output( suppress_example_values: bool = ..., outcome: str | None = ..., ) -> str: ... -def render_diff_json_output( - *, - result: DiffExecutionResult, - from_label: str, - to_label: str, - max_value_length: int | None = ..., - suppress_example_values: bool = ..., - phase_seconds: dict[str, float] | None = ..., - outcome: str | None = ..., -) -> str: ... def render_diff_error_json( *, status: str, @@ -1556,20 +1547,10 @@ def render_diff_error_json( help_text: str | None = ..., size_guard: dict[str, object] | None = ..., ) -> str: ... -def write_diff_output( - *, - request: DiffCommandRequest, - from_label: str, - to_label: str, - max_column_examples: int, - max_row_only_examples: int, - result: DiffExecutionResult, - phase_seconds: dict[str, float] | None = ..., - outcome: str | None = ..., -) -> None: ... -def resolve_diff_outcome( - *, result: DiffExecutionResult, query_row_comparison: bool = ... -) -> tuple[str, int]: ... +def run_diff_command(*, request: DiffCommandRequest) -> int: ... +def execute_query_diff( + *, request: DiffCommandRequest, preparation: QueryDiffPreparation +) -> DiffExecutionResult: ... def machine_output_mode(args: CliNamespace, /) -> bool: ... def current_native_progress_projector() -> NativeProgressProjector | None: ... diff --git a/src/sqlbuild/cli/commands/_helpers/diff/__init__.py b/src/sqlbuild/cli/commands/_helpers/diff/__init__.py deleted file mode 100644 index b12af6e04c..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Diff command helper package.""" diff --git a/src/sqlbuild/cli/commands/_helpers/diff/execution.py b/src/sqlbuild/cli/commands/_helpers/diff/execution.py deleted file mode 100644 index 6e0381fa28..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/execution.py +++ /dev/null @@ -1,685 +0,0 @@ -"""Diff command execution phases.""" - -from __future__ import annotations - -import sys -import time -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - resolve_diff_outcome, - resolve_project_connection_config, - resolve_target_connection_config, - run_diff_pipeline, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.cli.commands._helpers.diff.size_guard import ( - build_diff_size_guard_error, - resolve_full_diff_size_limits, -) -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.constants import READ_ONLY_QUERY_ROOT_KEYS -from sqlbuild.cli.commands.exceptions import CliUserError, QueryDiffExecutionError -from sqlbuild.cli.commands.models import ( - DiffCommandRequest, - DiffInvocation, - DirectDiffPreparation, - QueryDiffPreparation, - QueryDiffRunOutcome, -) -from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config -from sqlbuild.compiler.compile.main.effective_target_namespace import ( - build_effective_target_namespace, -) -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle -from sqlbuild.executor.diff.exceptions import FullDiffSizeGuardError -from sqlbuild.executor.diff.main.config import parse_cli_tolerance_overrides -from sqlbuild.executor.diff.main.execute import execute_diff -from sqlbuild.executor.diff.main.execute_query import ( - execute_query_diff as execute_prepared_query_diff, -) -from sqlbuild.executor.diff.models import ( - DiffExecutionOptions, - DiffExecutionResult, - ModelDiffResult, - QueryDiffArtifact, - QueryDiffArtifactCleanupResult, - RowDiffSamplingOverride, -) -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name - - -def prepare_direct_diff( - *, request: DiffCommandRequest, invocation: DiffInvocation -) -> DirectDiffPreparation: - """Resolve direct target diff adapter, compiled projects, and limits.""" - - if request.from_name is None: - raise CliUserError("model diff requires FROM or FROM:TO", code="C224") - from_target: str = request.from_name - to_target: str = _resolve_diff_to_target( - request=request, invocation=invocation, from_target=from_target - ) - if from_target not in invocation.discovered_inputs.project_config.targets: - raise CliUserError(f"unknown diff FROM target '{from_target}'", code="C205") - if to_target not in invocation.discovered_inputs.project_config.targets: - raise CliUserError(f"unknown diff TO target '{to_target}'", code="C206") - effective_adapter_name: str = resolve_effective_adapter_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=effective_adapter_name, project_dir=invocation.effective_project_dir - ) - connection_config: dict[str, object] = resolve_target_connection_config( - discovered_inputs=invocation.discovered_inputs, - project_dir=invocation.effective_project_dir, - target_name=to_target, - cli_vars=request.cli_vars, - ) - left_project: Any - right_project: Any - selected_names: tuple[str, ...] - left_project, right_project, selected_names = run_diff_pipeline( - discovered_inputs=invocation.discovered_inputs, - adapter=adapter, - from_target=from_target, - to_target=to_target, - resolved_connection=connection_config, - no_sql_validation=request.no_sql_validation, - select=request.select, - exclude=request.exclude, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ) - if not selected_names: - raise CliUserError("no diffable models found in the selected scope", code="C207") - return DirectDiffPreparation( - from_target=from_target, - to_target=to_target, - adapter=adapter, - left_project=left_project, - right_project=right_project, - selected_names=selected_names, - connection_config=connection_config, - effective_max_column_examples=_effective_max_examples( - explicit_value=request.max_column_examples, verbose=request.verbose - ), - effective_max_row_only_examples=_effective_max_examples( - explicit_value=request.max_row_only_examples, verbose=request.verbose - ), - full_size_limits=resolve_full_diff_size_limits( - request=request, - discovered_inputs=invocation.discovered_inputs, - from_target=from_target, - to_target=to_target, - ), - ) - - -def _resolve_diff_to_target( - *, request: DiffCommandRequest, invocation: DiffInvocation, from_target: str -) -> str: - if request.to_name is not None: - return request.to_name - active_target: str | None = resolve_target_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - selected_target=None, - ) - if active_target is None: - raise CliUserError( - "model diff requires FROM:TO when the project has no active target", - code="C271", - help="set default_target, or pass both targets as FROM:TO", - ) - if active_target == from_target: - raise CliUserError( - f"diff FROM '{from_target}' is the active target, so there is nothing to compare", - code="C272", - help="pass both targets as FROM:TO", - ) - return active_target - - -def execute_direct_diff( - *, request: DiffCommandRequest, preparation: DirectDiffPreparation -) -> DiffExecutionResult: - """Execute a direct target-to-target diff.""" - - connection: Any = preparation.adapter.connect(preparation.connection_config) - try: - return execute_diff( - adapter=preparation.adapter, - connection=connection, - left_project=preparation.left_project, - right_project=preparation.right_project, - selected_names=preparation.selected_names, - options=DiffExecutionOptions( - schema_only=request.schema_only, - bounded=request.bounded, - collect_samples=not request.schema_only, - max_column_examples=preparation.effective_max_column_examples, - max_row_only_examples=preparation.effective_max_row_only_examples, - max_models=request.max_models, - max_columns=request.max_columns, - sampling_override=_sampling_override(request=request), - unique_key_override=request.unique_key_override, - unkeyed=request.unkeyed, - excluded_columns_override=request.excluded_columns_override, - tolerance_overrides=parse_cli_tolerance_overrides( - values=request.tolerance_overrides - ), - full_size_limits=preparation.full_size_limits, - ), - ) - except FullDiffSizeGuardError as error: - raise build_diff_size_guard_error( - request=request, - from_target=preparation.from_target, - to_target=preparation.to_target, - error=error, - ) from error - finally: - preparation.adapter.close(connection) - - -def prepare_query_diff( - *, request: DiffCommandRequest, invocation: DiffInvocation -) -> QueryDiffPreparation: - """Resolve raw SQL, active target namespace, adapter, and connection.""" - - effective_adapter_name: str = resolve_effective_adapter_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=effective_adapter_name, - project_dir=invocation.effective_project_dir, - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=invocation.discovered_inputs, - project_dir=invocation.effective_project_dir, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - connection_database: object | None = connection_config.get("database") - if effective_adapter_name in { - BuiltinAdapter.DUCKDB.value, - BuiltinAdapter.MOTHERDUCK.value, - }: - connection_database = None - target_name, _target, database, schema = build_effective_target_namespace( - discovered_inputs=invocation.discovered_inputs, - selected_target=request.selected_target, - connection_database=connection_database, - connection_schema=connection_config.get("schema"), - default_database=adapter.default_database(), - default_schema=adapter.default_schema(), - ) - if schema is None: - raise CliUserError( - "raw-query diff requires an active target schema", - code="C227", - help="configure the selected target schema before running the comparison", - ) - left_sql: str = _resolve_query_input( - inline=request.left_query, - file=request.left_query_file, - label="left", - ) - right_sql: str = _resolve_query_input( - inline=request.right_query, - file=request.right_query_file, - label="right", - ) - left_sql = _validate_read_only_query(sql=left_sql, label="left", adapter=adapter) - right_sql = _validate_read_only_query(sql=right_sql, label="right", adapter=adapter) - _runtime_target, _effective_vars, run_id = build_effective_runtime_config( - discovered_inputs=invocation.discovered_inputs, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - return QueryDiffPreparation( - adapter=adapter, - connection_config=connection_config, - left_sql=left_sql, - right_sql=right_sql, - left_label=(request.left_label or "left query").strip(), - right_label=(request.right_label or "right query").strip(), - selected_target=target_name, - database=database, - schema=schema, - run_id=run_id, - artifact_ttl=invocation.discovered_inputs.project_config.diff.query_artifact_ttl, - effective_max_column_examples=_effective_max_examples( - explicit_value=request.max_column_examples, - verbose=request.verbose, - ), - effective_max_row_only_examples=_effective_max_examples( - explicit_value=request.max_row_only_examples, - verbose=request.verbose, - ), - ) - - -def execute_query_diff( - *, request: DiffCommandRequest, preparation: QueryDiffPreparation -) -> QueryDiffRunOutcome: - """Materialize, compare, and clean one raw-query pair.""" - - adapter: BaseAdapter = preparation.adapter - total_started_at: float = time.perf_counter() - phase_seconds: dict[str, float] = {} - print("Query diff connection START", file=sys.stderr) - connection_started_at: float = time.perf_counter() - try: - connection: Any = adapter.connect(preparation.connection_config) - except Exception: - connection_elapsed: float = time.perf_counter() - connection_started_at - print(f"Query diff connection ERROR ({connection_elapsed:.2f}s)", file=sys.stderr) - print( - f"Query diff total ERROR ({time.perf_counter() - total_started_at:.2f}s)", - file=sys.stderr, - ) - raise - phase_seconds["connection"] = time.perf_counter() - connection_started_at - print(f"Query diff connection OK ({phase_seconds['connection']:.2f}s)", file=sys.stderr) - left, right = _build_query_diff_artifacts(adapter=adapter, preparation=preparation) - created: list[QueryDiffArtifact] = [] - cleanup_errors: list[Exception] = [] - model_result: ModelDiffResult | None = None - try: - phase_seconds["reconciliation"] = _reconcile_query_diff_artifacts( - adapter=adapter, - connection=connection, - preparation=preparation, - ) - expires_at, phase_seconds["inspection"] = _inspect_query_diff_inputs( - adapter=adapter, - connection=connection, - preparation=preparation, - artifacts=(left, right), - max_columns=request.max_columns, - ) - print("Query diff materialization START", file=sys.stderr) - materialization_started_at: float = time.perf_counter() - try: - QueryDiffArtifactLifecycle.materialize( - adapter=adapter, - connection=connection, - artifact=left, - sql=preparation.left_sql, - expires_at=expires_at, - ) - created.append(left) - QueryDiffArtifactLifecycle.materialize( - adapter=adapter, - connection=connection, - artifact=right, - sql=preparation.right_sql, - expires_at=expires_at, - ) - created.append(right) - except Exception: - phase_seconds["materialization"] = time.perf_counter() - materialization_started_at - print( - f"Query diff materialization ERROR ({phase_seconds['materialization']:.2f}s)", - file=sys.stderr, - ) - raise - phase_seconds["materialization"] = time.perf_counter() - materialization_started_at - print( - f"Query diff materialization OK ({preparation.schema}; expires " - f"{expires_at.isoformat()}; {phase_seconds['materialization']:.2f}s)", - file=sys.stderr, - ) - print("Query diff comparison START", file=sys.stderr) - comparison_started_at: float = time.perf_counter() - try: - model_result = execute_prepared_query_diff( - adapter=adapter, - connection=connection, - left_relation=left.relation, - right_relation=right.relation, - options=DiffExecutionOptions( - schema_only=request.schema_only, - collect_samples=not request.schema_only, - max_column_examples=preparation.effective_max_column_examples, - max_row_only_examples=preparation.effective_max_row_only_examples, - max_columns=request.max_columns, - sampling_override=_sampling_override(request=request), - unique_key_override=request.unique_key_override, - unkeyed=request.unkeyed, - excluded_columns_override=request.excluded_columns_override, - tolerance_overrides=parse_cli_tolerance_overrides( - values=request.tolerance_overrides - ), - comparison_name=(f"{preparation.left_label} vs {preparation.right_label}"), - ), - ) - except Exception: - phase_seconds["comparison"] = time.perf_counter() - comparison_started_at - print( - f"Query diff comparison ERROR ({phase_seconds['comparison']:.2f}s)", - file=sys.stderr, - ) - raise - phase_seconds["comparison"] = time.perf_counter() - comparison_started_at - print(f"Query diff comparison OK ({phase_seconds['comparison']:.2f}s)", file=sys.stderr) - finally: - active_error: BaseException | None = sys.exc_info()[1] - cleanup_errors, phase_seconds["cleanup"] = _cleanup_query_diff_artifacts( - adapter=adapter, - connection=connection, - created=created, - ) - phase_seconds["total"] = time.perf_counter() - total_started_at - total_status: str = "ERROR" if active_error is not None or cleanup_errors else "OK" - print( - f"Query diff total {total_status} ({phase_seconds['total']:.2f}s)", - file=sys.stderr, - ) - if cleanup_errors: - raise cleanup_errors[0] - if model_result is None: - raise QueryDiffExecutionError( - "query diff comparison completed without a result", code="C256" - ) - result: DiffExecutionResult = DiffExecutionResult(model_results=(model_result,)) - outcome, exit_code = resolve_diff_outcome( - result=result, query_row_comparison=not request.schema_only - ) - return QueryDiffRunOutcome( - result=result, - phase_seconds=phase_seconds, - outcome=outcome, - exit_code=exit_code, - ) - - -def _build_query_diff_artifacts( - *, adapter: BaseAdapter, preparation: QueryDiffPreparation -) -> tuple[QueryDiffArtifact, QueryDiffArtifact]: - return ( - QueryDiffArtifactLifecycle.build( - adapter=adapter, - run_id=preparation.run_id, - side="left", - database=preparation.database, - schema=preparation.schema, - ), - QueryDiffArtifactLifecycle.build( - adapter=adapter, - run_id=preparation.run_id, - side="right", - database=preparation.database, - schema=preparation.schema, - ), - ) - - -def _cleanup_query_diff_artifacts( - *, - adapter: BaseAdapter, - connection: Any, - created: list[QueryDiffArtifact], -) -> tuple[list[Exception], float]: - print("Query diff immediate cleanup START", file=sys.stderr) - started_at: float = time.perf_counter() - errors: list[Exception] = [] - for artifact in reversed(created): - try: - QueryDiffArtifactLifecycle.cleanup( - adapter=adapter, - connection=connection, - artifact=artifact, - ) - except Exception as error: - errors.append(error) - try: - adapter.close(connection) - except Exception as error: - errors.append(error) - elapsed: float = time.perf_counter() - started_at - if errors: - print( - f"Query diff immediate cleanup ERROR ({len(errors)} failed; {elapsed:.2f}s)", - file=sys.stderr, - ) - else: - print( - f"Query diff immediate cleanup OK ({len(created)} removed; {elapsed:.2f}s)", - file=sys.stderr, - ) - return errors, elapsed - - -def _inspect_query_diff_inputs( - *, - adapter: BaseAdapter, - connection: Any, - preparation: QueryDiffPreparation, - artifacts: tuple[QueryDiffArtifact, QueryDiffArtifact], - max_columns: int | None, -) -> tuple[datetime, float]: - print("Query diff inspection START", file=sys.stderr) - started_at: float = time.perf_counter() - try: - artifact_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple( - (artifact.database, artifact.schema, artifact.name) for artifact in artifacts - ), - ) - for artifact in artifacts: - if artifact_lookup.exists( - database=artifact.database, - schema=artifact.schema, - name=artifact.name, - ): - raise CliUserError( - f"query-diff artifact already exists: {artifact.relation}", - code="C245", - help="wait for its TTL cleanup or investigate the reserved-name relation", - ) - ttl: Duration | None = Duration.parse(preparation.artifact_ttl) - if ttl is None: - raise CliUserError("diff.query_artifact_ttl must be a positive duration", code="C228") - _preflight_query_columns( - adapter=adapter, - connection=connection, - sql=preparation.left_sql, - label=preparation.left_label, - max_columns=max_columns, - ) - _preflight_query_columns( - adapter=adapter, - connection=connection, - sql=preparation.right_sql, - label=preparation.right_label, - max_columns=max_columns, - ) - except Exception: - elapsed: float = time.perf_counter() - started_at - print( - f"Query diff inspection ERROR ({elapsed:.2f}s)", - file=sys.stderr, - ) - raise - elapsed = time.perf_counter() - started_at - print(f"Query diff inspection OK ({elapsed:.2f}s)", file=sys.stderr) - return ttl.add_to(datetime.now(UTC)), elapsed - - -def _reconcile_query_diff_artifacts( - *, - adapter: BaseAdapter, - connection: Any, - preparation: QueryDiffPreparation, -) -> float: - print("Query diff reconciliation START", file=sys.stderr) - started_at: float = time.perf_counter() - try: - result: QueryDiffArtifactCleanupResult = QueryDiffArtifactLifecycle.cleanup_expired( - adapter=adapter, - connection=connection, - database=preparation.database, - schema=preparation.schema, - now=datetime.now(UTC), - ) - except Exception as error: - elapsed: float = time.perf_counter() - started_at - print( - "Query diff reconciliation WARNING recovery cleanup could not inspect historical " - f"ownership evidence: {error} ({elapsed:.2f}s)", - file=sys.stderr, - ) - return elapsed - elapsed = time.perf_counter() - started_at - print( - f"Query diff reconciliation OK ({len(result.cleaned)} expired; {elapsed:.2f}s)", - file=sys.stderr, - ) - for relation in result.untracked: - print( - "Query diff reconciliation WARNING reserved-name relation lacks matching " - f"ownership evidence: {relation.name}", - file=sys.stderr, - ) - return elapsed - - -def _resolve_query_input(*, inline: str | None, file: Path | None, label: str) -> str: - if inline is not None: - return inline - if file is None: - raise CliUserError(f"raw-query diff requires a {label} query", code="C229") - if not file.exists() or not file.is_file(): - raise CliUserError(f"{label} query file does not exist: {file}", code="C230") - try: - return file.read_text(encoding="utf-8") - except (OSError, UnicodeError) as error: - raise CliUserError( - f"{label} query file could not be read as UTF-8: {file}", code="C231" - ) from error - - -def _validate_read_only_query(*, sql: str, label: str, adapter: BaseAdapter) -> str: - normalized_sql: str = sql.strip() - if normalized_sql.endswith(";"): - normalized_sql = normalized_sql[:-1].rstrip() - if not normalized_sql: - raise CliUserError(f"{label} query must not be empty", code="C232") - polyglot: Any | None = import_polyglot_sql() - if polyglot is None: - raise CliUserError("raw-query diff requires the bundled SQL parser", code="C233") - try: - expression: Any = polyglot.parse_one( - normalized_sql, - dialect=adapter.sql_analysis_dialect() or "generic", - ) - except Exception as error: - raise CliUserError( - f"{label} query is not one valid SQL statement: {error}", code="C234" - ) from error - if str(getattr(expression, "key", "")).lower() not in READ_ONLY_QUERY_ROOT_KEYS: - raise CliUserError( - f"{label} query must be one read-only SELECT, WITH, or VALUES expression", - code="C235", - ) - mutating_keys: frozenset[str] = frozenset( - { - "alter", - "command", - "copy", - "create", - "delete", - "drop", - "execute", - "grant", - "insert", - "load_data", - "merge", - "pragma", - "revoke", - "set", - "transaction", - "truncate", - "uncache", - "update", - "use", - } - ) - if any(str(getattr(node, "key", "")).lower() in mutating_keys for node in expression.walk()): - raise CliUserError( - f"{label} query must not contain data-changing or administrative statements", - code="C240", - ) - expression_args: object = getattr(expression, "args", {}) - if isinstance(expression_args, dict) and ( - expression_args.get("into") is not None or bool(expression_args.get("locks")) - ): - raise CliUserError( - f"{label} query must not write a relation or acquire row locks", - code="C244", - ) - return normalized_sql - - -def _preflight_query_columns( - *, - adapter: BaseAdapter, - connection: Any, - sql: str, - label: str, - max_columns: int | None, -) -> None: - display_label: str = label if label.lower().endswith("query") else f"{label} query" - columns: tuple[str, ...] = adapter.query_column_names(connection=connection, sql=sql) - if not columns: - raise CliUserError(f"{display_label} returned no columns", code="C237") - normalized: tuple[str, ...] = tuple(column.lower() for column in columns) - duplicates: tuple[str, ...] = tuple( - sorted({column for column in normalized if normalized.count(column) > 1}) - ) - if duplicates: - raise CliUserError( - f"{display_label} returns duplicate column names: {', '.join(duplicates)}", - code="C238", - help="add unique aliases before comparing query results", - ) - if max_columns is not None and len(columns) > max_columns: - raise CliUserError( - f"{display_label} has {len(columns)} columns, exceeding --max-columns {max_columns}", - code="C239", - ) - - -def _effective_max_examples(*, explicit_value: int | None, verbose: bool) -> int: - return explicit_value if explicit_value is not None else (10 if verbose else 3) - - -def _sampling_override(*, request: DiffCommandRequest) -> RowDiffSamplingOverride: - return RowDiffSamplingOverride( - row_limit=request.sample_rows, - seed=request.sample_seed, - exhaustive=request.exhaustive, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/diff/invocation.py b/src/sqlbuild/cli/commands/_helpers/diff/invocation.py deleted file mode 100644 index fce0894b86..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/invocation.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Diff command invocation resolution and validation.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import discover_project_inputs -from sqlbuild.cli.commands.exceptions import CliUserError, QueryDiffIncompleteError -from sqlbuild.cli.commands.models import DiffCommandRequest, DiffInvocation -from sqlbuild.compiler.discovery.main.discover_configuration import ( - discover_project_configuration, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors - - -def resolve_diff_invocation(*, request: DiffCommandRequest) -> DiffInvocation: - """Validate diff flags and discover project inputs.""" - - try: - _validate_diff_request(request=request) - except CliUserError as error: - if is_query_diff_request(request=request): - raise QueryDiffIncompleteError( - error.message, - code=error.code, - help=error.help, - ) from error - raise - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = ( - discover_project_configuration(project_dir=effective_project_dir) - if is_query_diff_request(request=request) - else discover_project_inputs(project_dir=effective_project_dir) - ) - reject_unit_test_selectors( - select=request.select, exclude=request.exclude, discovered_inputs=discovered_inputs - ) - if not request.select and not is_query_diff_request(request=request): - raise CliUserError("diff requires --select in v1", code="C204") - return DiffInvocation( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - ) - - -def _validate_diff_request(*, request: DiffCommandRequest) -> None: - query_mode: bool = is_query_diff_request(request=request) - selected_modes: int = ( - int(request.full) + int(request.schema_only) + int(request.bounded is not None) - ) - if selected_modes > 1: - raise CliUserError( - "diff accepts at most one of --full, --schema-only, or --bounded", - code="C201", - ) - if request.max_column_examples is not None and request.max_column_examples <= 0: - raise CliUserError("diff --max-column-examples must be positive", code="C202") - if request.max_row_only_examples is not None and request.max_row_only_examples <= 0: - raise CliUserError("diff --max-row-only-examples must be positive", code="C203") - if request.sample_rows is not None and request.sample_rows <= 0: - raise CliUserError("diff --sample-rows must be positive", code="C208") - if request.exhaustive and request.sample_rows is not None: - raise CliUserError("diff --exhaustive cannot be combined with --sample-rows", code="C209") - if request.max_models is not None and request.max_models <= 0: - raise CliUserError("diff --max-models must be positive", code="C210") - if request.max_columns is not None and request.max_columns <= 0: - raise CliUserError("diff --max-columns must be positive", code="C211") - if request.max_value_length is not None and request.max_value_length <= 0: - raise CliUserError("diff --max-value-length must be positive", code="C246") - if request.full_example_values and request.max_value_length is not None: - raise CliUserError( - "diff --full-example-values cannot be combined with --max-value-length", - code="C247", - ) - if request.full_example_values and request.suppress_example_values: - raise CliUserError( - "diff --full-example-values cannot be combined with --no-example-values", - code="C248", - ) - if request.unkeyed and request.unique_key_override: - raise CliUserError("diff --unkeyed cannot be combined with --key", code="C226") - if request.unkeyed and (request.sample_rows is not None or request.sample_seed is not None): - raise CliUserError("unkeyed diff does not support sampling", code="C222") - if request.unkeyed and request.tolerance_overrides: - raise CliUserError("unkeyed diff does not support tolerances", code="C223") - if any(not key.strip() for key in request.unique_key_override): - raise CliUserError("diff --key values must not be empty", code="C241") - normalized_keys: tuple[str, ...] = tuple(key.lower() for key in request.unique_key_override) - if len(set(normalized_keys)) != len(normalized_keys): - raise CliUserError("diff --key values must be unique", code="C242") - if query_mode: - _validate_query_diff_request(request=request) - else: - _validate_model_diff_request(request=request) - - -def is_query_diff_request(*, request: DiffCommandRequest) -> bool: - """Return whether any raw-query input was supplied.""" - - return any( - value is not None - for value in ( - request.left_query, - request.left_query_file, - request.right_query, - request.right_query_file, - ) - ) - - -def _validate_query_diff_request(*, request: DiffCommandRequest) -> None: - if request.from_name is not None or request.to_name is not None: - raise CliUserError("raw-query diff does not accept FROM:TO", code="C212") - if request.select or request.exclude: - raise CliUserError("raw-query diff does not accept model selectors", code="C213") - if request.bounded is not None: - raise CliUserError( - "raw-query diff does not support --bounded; filter both raw queries explicitly", - code="C214", - ) - if request.max_models is not None: - raise CliUserError("raw-query diff does not accept --max-models", code="C243") - if request.left_label is not None and not request.left_label.strip(): - raise CliUserError("raw-query diff --left-label must not be empty", code="C249") - if request.right_label is not None and not request.right_label.strip(): - raise CliUserError("raw-query diff --right-label must not be empty", code="C250") - if request.left_query is None and request.left_query_file is None: - raise CliUserError("raw-query diff requires a left query", code="C216") - if request.left_query is not None and request.left_query_file is not None: - raise CliUserError( - "raw-query diff accepts either --left-query or --left-query-file, not both", - code="C217", - ) - if request.right_query is None and request.right_query_file is None: - raise CliUserError("raw-query diff requires a right query", code="C218") - if request.right_query is not None and request.right_query_file is not None: - raise CliUserError( - "raw-query diff accepts either --right-query or --right-query-file, not both", - code="C219", - ) - if request.schema_only: - if request.unkeyed or request.unique_key_override: - raise CliUserError( - "schema-only raw-query diff does not accept --key or --unkeyed", - code="C220", - ) - return - if bool(request.unique_key_override) == request.unkeyed: - raise CliUserError( - "raw-query diff requires one or more --key values or explicit --unkeyed", - code="C221", - ) - - -def _validate_model_diff_request(*, request: DiffCommandRequest) -> None: - if request.from_name is None: - raise CliUserError("model diff requires FROM or FROM:TO", code="C224") - if request.selected_target is not None: - raise CliUserError("model diff does not accept --target", code="C225") - if request.left_label is not None or request.right_label is not None: - raise CliUserError("model diff does not accept query labels", code="C251") diff --git a/src/sqlbuild/cli/commands/_helpers/diff/size_guard.py b/src/sqlbuild/cli/commands/_helpers/diff/size_guard.py deleted file mode 100644 index bfb88e35fc..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/size_guard.py +++ /dev/null @@ -1,193 +0,0 @@ -"""Default full model diff size limits and guard error rendering.""" - -from __future__ import annotations - -import json -import shlex - -from sqlbuild.cli.commands.exceptions import DiffSizeGuardError -from sqlbuild.cli.commands.models import DiffCommandRequest -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.diff.exceptions import FullDiffSizeGuardError -from sqlbuild.executor.diff.models import FullDiffModelSize, FullDiffSideSize, FullDiffSizeLimits -from sqlbuild.spec.contracts.constants import DEFAULT_DIFF_MAX_FULL_ROWS -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config - -_BOUNDED_WINDOW_PLACEHOLDER: str = "" - - -def resolve_full_diff_size_limits( - *, - request: DiffCommandRequest, - discovered_inputs: DiscoveredProjectInputs, - from_target: str, - to_target: str, -) -> FullDiffSizeLimits | None: - """Return per-side row limits when no explicit comparison mode was requested.""" - - if request.full or request.schema_only or request.bounded is not None: - return None - return FullDiffSizeLimits( - left_target=from_target, - right_target=to_target, - left_max_rows=_target_max_full_rows( - discovered_inputs=discovered_inputs, target_name=from_target - ), - right_max_rows=_target_max_full_rows( - discovered_inputs=discovered_inputs, target_name=to_target - ), - ) - - -def build_diff_size_guard_error( - *, - request: DiffCommandRequest, - from_target: str, - to_target: str, - error: FullDiffSizeGuardError, -) -> DiffSizeGuardError: - """Describe every blocked model and the explicit commands that replace the guarded run.""" - - commands: dict[str, str | None] = _replacement_commands( - request=request, - from_target=from_target, - to_target=to_target, - blocked=error.blocked, - ) - lines: list[str] = [error.message] - model: FullDiffModelSize - for model in error.blocked: - lines.append( - f" {model.name}: {_side_text(model.left)}; {_side_text(model.right)}" - + ("" if model.has_cursor else "; no cursor") - ) - lines.append("Run one of these instead:") - lines.append(f" {commands['full']}") - if commands["bounded"] is not None: - lines.append(f" {commands['bounded']}") - lines.append(f" {commands['schema_only']}") - over_targets: tuple[str, ...] = _over_limit_targets(blocked=error.blocked) - bounded_help: str = ( - "" - if commands["bounded"] is None - else ", --bounded compares a recent cursor window " - f"(replace {_BOUNDED_WINDOW_PLACEHOLDER} with a duration such as 7d)" - ) - return DiffSizeGuardError( - "\n".join(lines), - code="C270", - help=( - f"--full compares every row{bounded_help}, and --schema-only compares columns " - "only; or raise " - + " and ".join(f"targets.{target}.diff.max_full_rows" for target in over_targets) - ), - details={ - "from_target": from_target, - "to_target": to_target, - "models": [_model_payload(model) for model in error.blocked], - "commands": commands, - }, - ) - - -def _over_limit_targets(*, blocked: tuple[FullDiffModelSize, ...]) -> tuple[str, ...]: - targets: dict[str, None] = {} - model: FullDiffModelSize - for model in blocked: - side: FullDiffSideSize - for side in (model.left, model.right): - if side.exceeds_limit: - targets[side.target] = None - return tuple(targets) - - -def _target_max_full_rows( - *, discovered_inputs: DiscoveredProjectInputs, target_name: str -) -> int | None: - configured: int | str | None = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ).diff.max_full_rows - if configured is None: - return DEFAULT_DIFF_MAX_FULL_ROWS - return configured if isinstance(configured, int) else None - - -def _side_text(side: FullDiffSideSize) -> str: - limit: str = "unlimited" if side.max_rows is None else f"{side.max_rows:,}" - if side.row_count is None: - size: str = "size unknown" if side.detail is None else f"size unknown: {side.detail}" - else: - size = f"{side.row_count:,} rows" - return f"{side.target} {size}, limit {limit}" - - -def _model_payload(model: FullDiffModelSize) -> dict[str, object]: - return { - "name": model.name, - "has_cursor": model.has_cursor, - "from": _side_payload(model.left), - "to": _side_payload(model.right), - } - - -def _side_payload(side: FullDiffSideSize) -> dict[str, object]: - return { - "target": side.target, - "relation": side.relation, - "row_count": side.row_count, - "max_full_rows": side.max_rows, - "exceeds_limit": side.exceeds_limit, - "detail": side.detail, - } - - -def _replacement_commands( - *, - request: DiffCommandRequest, - from_target: str, - to_target: str, - blocked: tuple[FullDiffModelSize, ...], -) -> dict[str, str | None]: - scope: tuple[str, ...] = _scope_arguments(request=request) - project: tuple[str, ...] = ( - () if request.project_dir is None else ("--project-dir", str(request.project_dir)) - ) - prefix: tuple[str, ...] = ("sqb", *project, "diff", f"{from_target}:{to_target}") - bounded_available: bool = all(model.has_cursor for model in blocked) - return { - "full": _join((*prefix, "--full", *scope)), - "bounded": ( - _join((*prefix, "--bounded", _BOUNDED_WINDOW_PLACEHOLDER, *scope)) - if bounded_available - else None - ), - "schema_only": _join((*prefix, "--schema-only", *scope)), - } - - -def _scope_arguments(*, request: DiffCommandRequest) -> tuple[str, ...]: - arguments: list[str] = [] - if request.select: - arguments.extend(("--select", *request.select)) - if request.exclude: - arguments.extend(("--exclude", *request.exclude)) - if request.unique_key_override: - arguments.extend(("--key", *request.unique_key_override)) - if request.unkeyed: - arguments.append("--unkeyed") - if request.excluded_columns_override: - arguments.extend(("--exclude-column", *request.excluded_columns_override)) - tolerance: str - for tolerance in request.tolerance_overrides: - arguments.extend(("--tolerance", tolerance)) - if request.cli_vars: - arguments.extend(("--vars", json.dumps(request.cli_vars, sort_keys=True))) - return tuple(arguments) - - -def _join(parts: tuple[str, ...]) -> str: - return " ".join( - part if part == _BOUNDED_WINDOW_PLACEHOLDER else shlex.quote(part) for part in parts - ) diff --git a/src/sqlbuild/cli/commands/_helpers/diff/validation.py b/src/sqlbuild/cli/commands/_helpers/diff/validation.py deleted file mode 100644 index 8320f2f72f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/validation.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Validation helpers for diff command arguments.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.exceptions import CliUserError - - -def parse_diff_name_range(name_range: str | None) -> tuple[str, str | None]: - """Parse a diff name range in FROM:TO or FROM form; a bare FROM leaves TO unset.""" - - if name_range is None: - raise CliUserError("diff requires FROM:TO", code="C208") - from_name: str - separator: str - to_name: str - from_name, separator, to_name = name_range.partition(":") - if not from_name or (separator and (not to_name or separator in to_name)): - raise CliUserError("diff range must be FROM:TO or FROM", code="C209") - return from_name, to_name or None diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index ac9c101da2..03e718d228 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -65,12 +65,6 @@ SCOPE_GLOBAL_SUMMARY: str = "summary" SCOPE_GLOBAL_ALL: str = "all" SCOPE_DEFAULT_PAGE_SIZE: int = 100 -READ_ONLY_QUERY_ROOT_KEYS: frozenset[str] = frozenset( - {"except", "intersect", "select", "union", "values"} -) -QUERY_DIFF_INCOMPLETE_PREPARATION_CODES: frozenset[str] = frozenset( - {"C229", "C230", "C231", "C232", "C233", "C234", "C235", "C240", "C244"} -) MISSING_ORIGIN_BUILD_HELP: str = ( "Pass --allow-missing-migration-origin to confirm in non-interactive runs." ) @@ -105,7 +99,6 @@ ) EXECUTION_LIMIT_BUILD_NOTE: str = "No warehouse changes were made." EXECUTION_LIMIT_DBT_NOTE: str = "dbt work has already run; no SQLBuild models were built." -QUERY_DIFF_INCOMPLETE_EXECUTION_CODES: frozenset[str] = frozenset({"C237", "C238", "C239"}) CLI_WAREHOUSE_SOURCE: str = "--warehouse" CONNECTION_WAREHOUSE_SOURCE: str = "connection warehouse" CONNECTION_DEFAULT_WAREHOUSE_SOURCE: str = "connection default" diff --git a/src/sqlbuild/cli/commands/main/inspection/_diff.py b/src/sqlbuild/cli/commands/main/inspection/_diff.py deleted file mode 100644 index be19e22119..0000000000 --- a/src/sqlbuild/cli/commands/main/inspection/_diff.py +++ /dev/null @@ -1,139 +0,0 @@ -"""CLI diff command entry point.""" - -from __future__ import annotations - -from sqlbuild._native import resolve_diff_outcome, write_diff_output -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.cli.commands._helpers.diff.execution import ( - execute_direct_diff, - execute_query_diff, - prepare_direct_diff, - prepare_query_diff, -) -from sqlbuild.cli.commands._helpers.diff.invocation import ( - is_query_diff_request, - resolve_diff_invocation, -) -from sqlbuild.cli.commands.constants import ( - QUERY_DIFF_INCOMPLETE_EXECUTION_CODES, - QUERY_DIFF_INCOMPLETE_PREPARATION_CODES, -) -from sqlbuild.cli.commands.exceptions import ( - CliUserError, - QueryDiffExecutionError, - QueryDiffIncompleteError, - QueryDiffOutcomeError, -) -from sqlbuild.cli.commands.models import ( - DiffCommandRequest, - DiffInvocation, - DirectDiffPreparation, - QueryDiffPreparation, - QueryDiffRunOutcome, -) -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.diff.constants import ( - ROW_DIFF_KEY_ERROR_FRAGMENT, - ROW_DIFF_KEY_ERROR_PREFIX, -) -from sqlbuild.executor.diff.models import DiffExecutionResult - - -def run_diff(request: DiffCommandRequest) -> int: - """Execute the diff command.""" - - query_mode: bool = is_query_diff_request(request=request) - try: - invocation: DiffInvocation = resolve_diff_invocation(request=request) - except QueryDiffOutcomeError: - raise - except Exception as error: - if query_mode: - raise QueryDiffExecutionError(str(error), code="C252") from error - raise - if query_mode: - try: - query_preparation: QueryDiffPreparation = prepare_query_diff( - request=request, - invocation=invocation, - ) - except CliUserError as error: - error_type: type[QueryDiffOutcomeError] = ( - QueryDiffIncompleteError - if error.code in QUERY_DIFF_INCOMPLETE_PREPARATION_CODES - else QueryDiffExecutionError - ) - raise error_type(error.message, code=error.code, help=error.help) from error - except Exception as error: - raise QueryDiffExecutionError(str(error), code="C253") from error - try: - query_outcome: QueryDiffRunOutcome = execute_query_diff( - request=request, - preparation=query_preparation, - ) - except QueryDiffOutcomeError: - raise - except ExecutorInputError as error: - raise QueryDiffIncompleteError( - str(error), - code=str(getattr(error, "code", "C254")), - help=getattr(error, "help", None), - ) from error - except CliUserError as error: - error_type = ( - QueryDiffIncompleteError - if error.code in QUERY_DIFF_INCOMPLETE_EXECUTION_CODES - else QueryDiffExecutionError - ) - raise error_type(error.message, code=error.code, help=error.help) from error - except AdapterUserError as error: - error_type = ( - QueryDiffIncompleteError - if _is_query_key_preflight_error(error=error) - else QueryDiffExecutionError - ) - raise error_type(error.message, code=error.code, help=error.help) from error - except Exception as error: - raise QueryDiffExecutionError(str(error), code="C255") from error - try: - write_diff_output( - request=request, - from_label=query_preparation.left_label, - to_label=query_preparation.right_label, - max_column_examples=query_preparation.effective_max_column_examples, - max_row_only_examples=query_preparation.effective_max_row_only_examples, - result=query_outcome.result, - phase_seconds=query_outcome.phase_seconds, - outcome=query_outcome.outcome, - ) - except Exception as error: - raise QueryDiffExecutionError( - f"failed to publish query diff output: {error}", code="C257" - ) from error - return query_outcome.exit_code - - direct_preparation: DirectDiffPreparation = prepare_direct_diff( - request=request, - invocation=invocation, - ) - result: DiffExecutionResult = execute_direct_diff( - request=request, - preparation=direct_preparation, - ) - write_diff_output( - request=request, - from_label=direct_preparation.from_target, - to_label=direct_preparation.to_target, - max_column_examples=direct_preparation.effective_max_column_examples, - max_row_only_examples=direct_preparation.effective_max_row_only_examples, - result=result, - ) - exit_code: int - _outcome, exit_code = resolve_diff_outcome(result=result) - return exit_code - - -def _is_query_key_preflight_error(*, error: AdapterUserError) -> bool: - return error.message.startswith(ROW_DIFF_KEY_ERROR_PREFIX) and ( - ROW_DIFF_KEY_ERROR_FRAGMENT in error.message - ) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 4129431c76..8d66df4b7c 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Any, TextIO +from typing import TYPE_CHECKING, TextIO from sqlbuild.cli.commands.types import ( PlaygroundTemplate, @@ -62,7 +62,6 @@ from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput from sqlbuild.compiler.refactoring.models import RefactorProject from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult - from sqlbuild.executor.diff.models import DiffExecutionResult, FullDiffSizeLimits from sqlbuild.executor.janitor.models import JanitorPlan from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.integrations.dbt.models import DbtInitRequest @@ -407,30 +406,6 @@ class DiffCommandRequest: full_example_values: bool = False -@dataclass(frozen=True) -class DiffInvocation: - """Resolved project discovery and mode for diff.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - - -@dataclass(frozen=True) -class DirectDiffPreparation: - """Resolved direct diff adapter and compiled target projects.""" - - from_target: str - to_target: str - adapter: BaseAdapter - left_project: Any - right_project: Any - selected_names: tuple[str, ...] - connection_config: dict[str, object] - effective_max_column_examples: int - effective_max_row_only_examples: int - full_size_limits: FullDiffSizeLimits | None = None - - @dataclass(frozen=True) class QueryDiffPreparation: """Resolved raw-query diff adapter, connection, SQL, and limits.""" @@ -450,16 +425,6 @@ class QueryDiffPreparation: effective_max_row_only_examples: int -@dataclass(frozen=True) -class QueryDiffRunOutcome: - """Raw-query result plus measured lifecycle phase durations.""" - - result: DiffExecutionResult - phase_seconds: dict[str, float] - outcome: str - exit_code: int - - @dataclass(frozen=True) class FreshnessCommandRequest: """CLI inputs for one `sqb freshness` invocation.""" diff --git a/src/sqlbuild/compiler/compile/main/effective_target_namespace.py b/src/sqlbuild/compiler/compile/main/_effective_target_namespace.py similarity index 100% rename from src/sqlbuild/compiler/compile/main/effective_target_namespace.py rename to src/sqlbuild/compiler/compile/main/_effective_target_namespace.py diff --git a/src/sqlbuild/compiler/discovery/main/discover_configuration.py b/src/sqlbuild/compiler/discovery/main/_discover_configuration.py similarity index 100% rename from src/sqlbuild/compiler/discovery/main/discover_configuration.py rename to src/sqlbuild/compiler/discovery/main/_discover_configuration.py diff --git a/src/sqlbuild/executor/diff/constants.py b/src/sqlbuild/executor/diff/constants.py index 320cbd10ed..5387092604 100644 --- a/src/sqlbuild/executor/diff/constants.py +++ b/src/sqlbuild/executor/diff/constants.py @@ -4,8 +4,6 @@ ROW_DIFF_RELATIVE_TOLERANCE_KEY: str = "relative" DIFF_INPUT_KIND_MODEL: str = "model" DIFF_INPUT_KIND_QUERY: str = "query" -ROW_DIFF_KEY_ERROR_PREFIX: str = "row diff " -ROW_DIFF_KEY_ERROR_FRAGMENT: str = " unique_key values" ROW_DIFF_TOLERANCE_KEYS: frozenset[str] = frozenset( (ROW_DIFF_ABSOLUTE_TOLERANCE_KEY, ROW_DIFF_RELATIVE_TOLERANCE_KEY) ) diff --git a/src/sqlbuild/executor/diff/main/config.py b/src/sqlbuild/executor/diff/main/_config.py similarity index 100% rename from src/sqlbuild/executor/diff/main/config.py rename to src/sqlbuild/executor/diff/main/_config.py diff --git a/src/sqlbuild/executor/diff/main/execute_query.py b/src/sqlbuild/executor/diff/main/_execute_query.py similarity index 100% rename from src/sqlbuild/executor/diff/main/execute_query.py rename to src/sqlbuild/executor/diff/main/_execute_query.py diff --git a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py index f0a8e3f9af..04d94ba4fc 100644 --- a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py +++ b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py @@ -9,10 +9,10 @@ import duckdb import pytest +from sqlbuild._native import execute_query_diff from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands._helpers.diff.execution import execute_query_diff from sqlbuild.cli.commands.models import DiffCommandRequest, QueryDiffPreparation from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py index 676b13d528..8df69e144c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py @@ -1,8 +1,6 @@ from dataclasses import dataclass -from pathlib import Path -from typing import Literal -from sqlbuild.executor.diff.models import DiffExecutionResult, FullDiffSizeLimits +from sqlbuild.executor.diff.models import DiffExecutionResult @dataclass(frozen=True) @@ -16,41 +14,3 @@ class RenderDiffOutputTestCase: max_column_examples: int max_row_only_examples: int expected_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class FullDiffSizeLimitsTestCase: - description: str - full: bool - schema_only: bool - bounded: str | None - prod_max_full_rows: int | Literal["unlimited"] | None - dev_max_full_rows: int | Literal["unlimited"] | None - expected_limits: FullDiffSizeLimits | None - - -@dataclass(frozen=True) -class DiffSizeGuardErrorTestCase: - description: str - select: tuple[str, ...] - unique_key_override: tuple[str, ...] - has_cursor: bool - expected_full_command: str - expected_bounded_command: str | None - expected_message_fragments: tuple[str, ...] - expected_absent_fragments: tuple[str, ...] = () - project_dir: Path | None = None - - -@dataclass(frozen=True) -class ParseDiffNameRangeTestCase: - description: str - name_range: str - expected_result: tuple[str, str | None] - - -@dataclass(frozen=True) -class ParseDiffNameRangeErrorTestCase: - description: str - name_range: str | None - expected_code: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/helpers.py deleted file mode 100644 index 5767145518..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/helpers.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Shared builders for diff CLI helper unit tests.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Literal - -from sqlbuild.cli.commands.models import DiffCommandRequest -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.diff.models import FullDiffModelSize, FullDiffSideSize -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - TargetConfig, - TargetDiffConfig, -) - - -def diff_request( - *, - full: bool = False, - schema_only: bool = False, - bounded: str | None = None, - select: tuple[str, ...] = ("orders",), - unique_key_override: tuple[str, ...] = (), - project_dir: Path | None = None, -) -> DiffCommandRequest: - """Return a model diff request from prod with the TO target left to the default.""" - - return DiffCommandRequest( - project_dir=project_dir, - no_color=True, - no_sql_validation=False, - from_name="prod", - to_name=None, - full=full, - schema_only=schema_only, - bounded=bounded, - select=select, - unique_key_override=unique_key_override, - ) - - -def prod_dev_inputs( - *, - prod_max_full_rows: int | Literal["unlimited"] | None, - dev_max_full_rows: int | Literal["unlimited"] | None, -) -> DiscoveredProjectInputs: - """Return discovered inputs with prod and dev targets carrying the given diff limits.""" - - return DiscoveredProjectInputs( - project_config=ProjectConfig( - name="shop", - adapter="duckdb", - targets={ - "prod": TargetConfig( - schema="prod", diff=TargetDiffConfig(max_full_rows=prod_max_full_rows) - ), - "dev": TargetConfig( - schema="dev", diff=TargetDiffConfig(max_full_rows=dev_max_full_rows) - ), - }, - ), - local_config=LocalConfig(), - ) - - -def blocked_orders(*, has_cursor: bool) -> tuple[FullDiffModelSize, ...]: - """Return one blocked model: prod over its limit and dev a view of unknown size.""" - - return ( - FullDiffModelSize( - name="orders", - left=FullDiffSideSize( - target="prod", relation="prod.orders", row_count=12_500_000, max_rows=10_000_000 - ), - right=FullDiffSideSize( - target="dev", - relation="dev.orders", - row_count=None, - max_rows=10_000_000, - detail="no row count (view)", - ), - has_cursor=has_cursor, - ), - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_size_guard.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_size_guard.py deleted file mode 100644 index fb4604316d..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_size_guard.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Default full diff limits and size guard error rendering.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, cast - -import pytest - -from sqlbuild.cli.commands._helpers.diff.size_guard import ( - build_diff_size_guard_error, - resolve_full_diff_size_limits, -) -from sqlbuild.cli.commands.exceptions import DiffSizeGuardError -from sqlbuild.executor.diff.exceptions import FullDiffSizeGuardError -from sqlbuild.executor.diff.models import FullDiffSizeLimits -from tests.unit.src.sqlbuild.cli.commands._helpers.diff._test_types import ( - DiffSizeGuardErrorTestCase, - FullDiffSizeLimitsTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.diff.helpers import ( - blocked_orders, - diff_request, - prod_dev_inputs, -) - - -@pytest.mark.parametrize( - "test_case", - [ - FullDiffSizeLimitsTestCase( - description="unset limits use the built-in default", - full=False, - schema_only=False, - bounded=None, - prod_max_full_rows=None, - dev_max_full_rows=None, - expected_limits=FullDiffSizeLimits( - left_target="prod", - right_target="dev", - left_max_rows=10_000_000, - right_max_rows=10_000_000, - ), - ), - FullDiffSizeLimitsTestCase( - description="each side uses its own target limit", - full=False, - schema_only=False, - bounded=None, - prod_max_full_rows=500, - dev_max_full_rows="unlimited", - expected_limits=FullDiffSizeLimits( - left_target="prod", right_target="dev", left_max_rows=500, right_max_rows=None - ), - ), - FullDiffSizeLimitsTestCase( - description="explicit full bypasses the guard", - full=True, - schema_only=False, - bounded=None, - prod_max_full_rows=1, - dev_max_full_rows=1, - expected_limits=None, - ), - FullDiffSizeLimitsTestCase( - description="explicit schema only bypasses the guard", - full=False, - schema_only=True, - bounded=None, - prod_max_full_rows=1, - dev_max_full_rows=1, - expected_limits=None, - ), - FullDiffSizeLimitsTestCase( - description="explicit bounded bypasses the guard", - full=False, - schema_only=False, - bounded="7d", - prod_max_full_rows=1, - dev_max_full_rows=1, - expected_limits=None, - ), - ], - ids=lambda case: case.description, -) -def test_given_mode_flags_and_target_limits_when_resolving_then_guard_limits_match( - test_case: FullDiffSizeLimitsTestCase, -) -> None: - limits: FullDiffSizeLimits | None = resolve_full_diff_size_limits( - request=diff_request( - full=test_case.full, schema_only=test_case.schema_only, bounded=test_case.bounded - ), - discovered_inputs=prod_dev_inputs( - prod_max_full_rows=test_case.prod_max_full_rows, - dev_max_full_rows=test_case.dev_max_full_rows, - ), - from_target="prod", - to_target="dev", - ) - - assert limits == test_case.expected_limits - - -@pytest.mark.parametrize( - "test_case", - [ - DiffSizeGuardErrorTestCase( - description="model with cursor offers bounded command", - select=("orders",), - unique_key_override=(), - has_cursor=True, - expected_full_command="sqb diff prod:dev --full --select orders", - expected_bounded_command="sqb diff prod:dev --bounded --select orders", - expected_message_fragments=( - "orders: prod 12,500,000 rows, limit 10,000,000; " - "dev size unknown: no row count (view), limit 10,000,000", - "Run one of these instead:", - "sqb diff prod:dev --schema-only --select orders", - ), - ), - DiffSizeGuardErrorTestCase( - description="model without cursor omits bounded command and keeps key flags", - select=("tag:daily", "path:models/marts"), - unique_key_override=("order_id", "line number"), - has_cursor=False, - expected_full_command=( - "sqb diff prod:dev --full --select tag:daily path:models/marts " - "--key order_id 'line number'" - ), - expected_bounded_command=None, - expected_message_fragments=("; no cursor",), - expected_absent_fragments=("--bounded",), - ), - DiffSizeGuardErrorTestCase( - description="project dir is a global flag placed before diff", - select=("orders",), - unique_key_override=(), - has_cursor=False, - expected_full_command=( - "sqb --project-dir projects/shop diff prod:dev --full --select orders" - ), - expected_bounded_command=None, - expected_message_fragments=(), - project_dir=Path("projects/shop"), - ), - ], - ids=lambda case: case.description, -) -def test_given_blocked_models_when_building_guard_error_then_names_sizes_and_commands( - test_case: DiffSizeGuardErrorTestCase, -) -> None: - error: DiffSizeGuardError = build_diff_size_guard_error( - request=diff_request( - select=test_case.select, - unique_key_override=test_case.unique_key_override, - project_dir=test_case.project_dir, - ), - from_target="prod", - to_target="dev", - error=FullDiffSizeGuardError( - "full diff stopped", blocked=blocked_orders(has_cursor=test_case.has_cursor) - ), - ) - - assert error.code == "C270" - assert error.exit_code == 2 - assert error.status == "incomplete" - assert test_case.expected_full_command in error.message - fragment: str - for fragment in test_case.expected_message_fragments: - assert fragment in error.message - for fragment in test_case.expected_absent_fragments: - assert fragment not in error.message - assert fragment not in str(error.help) - commands: dict[str, Any] = cast(dict[str, Any], error.details["commands"]) - assert commands["full"] == test_case.expected_full_command - assert commands["bounded"] == test_case.expected_bounded_command - models: list[dict[str, Any]] = cast(list[dict[str, Any]], error.details["models"]) - assert models[0]["from"]["row_count"] == 12_500_000 - assert models[0]["to"]["row_count"] is None - assert models[0]["to"]["exceeds_limit"] is True - assert "targets.prod.diff.max_full_rows and targets.dev.diff.max_full_rows" in str(error.help) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_validation.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_validation.py deleted file mode 100644 index bc5290d355..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_validation.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Diff target range parsing.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.diff.validation import parse_diff_name_range -from sqlbuild.cli.commands.exceptions import CliUserError -from tests.unit.src.sqlbuild.cli.commands._helpers.diff._test_types import ( - ParseDiffNameRangeErrorTestCase, - ParseDiffNameRangeTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ParseDiffNameRangeTestCase( - description="explicit range keeps both targets", - name_range="prod:dev", - expected_result=("prod", "dev"), - ), - ParseDiffNameRangeTestCase( - description="bare target leaves TO for the active target", - name_range="prod", - expected_result=("prod", None), - ), - ], - ids=lambda case: case.description, -) -def test_given_target_range_when_parsing_then_returns_from_and_optional_to( - test_case: ParseDiffNameRangeTestCase, -) -> None: - assert parse_diff_name_range(test_case.name_range) == test_case.expected_result - - -@pytest.mark.parametrize( - "test_case", - [ - ParseDiffNameRangeErrorTestCase( - description="missing range", name_range=None, expected_code="C208" - ), - ParseDiffNameRangeErrorTestCase( - description="empty target", name_range="", expected_code="C209" - ), - ParseDiffNameRangeErrorTestCase( - description="empty TO", name_range="prod:", expected_code="C209" - ), - ParseDiffNameRangeErrorTestCase( - description="empty FROM", name_range=":dev", expected_code="C209" - ), - ParseDiffNameRangeErrorTestCase( - description="too many separators", name_range="a:b:c", expected_code="C209" - ), - ], - ids=lambda case: case.description, -) -def test_given_malformed_target_range_when_parsing_then_raises_coded_error( - test_case: ParseDiffNameRangeErrorTestCase, -) -> None: - with pytest.raises(CliUserError) as error_info: - parse_diff_name_range(test_case.name_range) - - assert error_info.value.code == test_case.expected_code - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) From b2885cdea333cad92e529dbd6148b18984e281a7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:51:13 +0100 Subject: [PATCH 34/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index fe3c81c39d..51cabb67a0 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 81100 +remaining_lines = 79857 [[stays]] path = "executor" From 192097b64d8de8c1b04c66eaf0d74eca15aec2c9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:50:47 +0100 Subject: [PATCH 35/37] feat: run sqb seed natively --- crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- .../_helpers/coded_error.rs | 9 +- .../src/cli_text/_helpers/mod.rs | 1 + .../src/cli_text/main/format_coded_error.rs | 14 + .../cli_text/main/format_tree_connector.rs | 10 + crates/sqlbuild-core/src/cli_text/main/mod.rs | 2 + crates/sqlbuild-core/src/lib.rs | 1 + .../src/seed_command/constants.rs | 6 + .../src/seed_command/main/mod.rs | 4 + .../main/render_seed_completion.rs | 41 ++ .../seed_command/main/render_seed_ready.rs | 24 + .../src/seed_command/main/render_seed_row.rs | 34 ++ .../src/seed_command/main/seed_exit_code.rs | 7 + crates/sqlbuild-core/src/seed_command/mod.rs | 8 + .../sqlbuild-core/src/seed_command/models.rs | 14 + .../sqlbuild-core/src/seed_command/tests.rs | 4 + .../seed_command/tests/test_seed_command.rs | 119 +++++ .../src/seed_command/tests/test_types.rs | 20 + .../src/test_command/_helpers/mod.rs | 1 - .../main/render_test_completion.rs | 6 +- crates/sqlbuild-core/src/test_command/mod.rs | 1 - .../src/bindings/_helpers/cli/registration.rs | 4 +- .../src/bindings/_helpers/mod.rs | 1 + .../bindings/_helpers/seed_command/command.rs | 493 ++++++++++++++++++ .../src/bindings/_helpers/seed_command/mod.rs | 2 + .../_helpers/seed_command/registration.rs | 9 + src/sqlbuild/_native.pyi | 2 + .../cli/commands/_helpers/seed/execution.py | 130 ----- .../cli/commands/_helpers/seed/invocation.py | 14 - .../cli/commands/_helpers/seed/outputs.py | 103 ---- .../cli/commands/_helpers/seed/planning.py | 107 ---- .../cli/commands/main/execution/_seed.py | 40 -- src/sqlbuild/cli/commands/models.py | 18 +- .../main/commands/{seed.py => _seed.py} | 0 .../{seed_state.py => _seed_state.py} | 0 .../presentation/main/surface_header.py | 12 - .../runtime/test_command_invocation.py | 7 +- .../main/test_static_command_planning.py | 4 +- 38 files changed, 840 insertions(+), 436 deletions(-) rename crates/sqlbuild-core/src/{test_command => cli_text}/_helpers/coded_error.rs (73%) create mode 100644 crates/sqlbuild-core/src/cli_text/main/format_coded_error.rs create mode 100644 crates/sqlbuild-core/src/cli_text/main/format_tree_connector.rs create mode 100644 crates/sqlbuild-core/src/seed_command/constants.rs create mode 100644 crates/sqlbuild-core/src/seed_command/main/mod.rs create mode 100644 crates/sqlbuild-core/src/seed_command/main/render_seed_completion.rs create mode 100644 crates/sqlbuild-core/src/seed_command/main/render_seed_ready.rs create mode 100644 crates/sqlbuild-core/src/seed_command/main/render_seed_row.rs create mode 100644 crates/sqlbuild-core/src/seed_command/main/seed_exit_code.rs create mode 100644 crates/sqlbuild-core/src/seed_command/mod.rs create mode 100644 crates/sqlbuild-core/src/seed_command/models.rs create mode 100644 crates/sqlbuild-core/src/seed_command/tests.rs create mode 100644 crates/sqlbuild-core/src/seed_command/tests/test_seed_command.rs create mode 100644 crates/sqlbuild-core/src/seed_command/tests/test_types.rs delete mode 100644 crates/sqlbuild-core/src/test_command/_helpers/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/seed_command/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/seed_command/registration.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/seed/execution.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/seed/invocation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/seed/outputs.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/seed/planning.py delete mode 100644 src/sqlbuild/cli/commands/main/execution/_seed.py rename src/sqlbuild/compiler/planner/main/commands/{seed.py => _seed.py} (100%) rename src/sqlbuild/compiler/planner/main/commands/{seed_state.py => _seed_state.py} (100%) delete mode 100644 src/sqlbuild/presentation/main/surface_header.py diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 6e185b6b5b..b46c6facef 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -332,9 +332,9 @@ const SEED_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const SEED_ROUTE: Route = routed( "run_seed", - ("sqlbuild.cli.commands.main.execution._seed", "run_seed"), + ("sqlbuild._native", "run_seed_command"), &SEED_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const LINEAGE_FIELDS: [Field; 11] = [ diff --git a/crates/sqlbuild-core/src/test_command/_helpers/coded_error.rs b/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs similarity index 73% rename from crates/sqlbuild-core/src/test_command/_helpers/coded_error.rs rename to crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs index 8b547ec171..b80362ec49 100644 --- a/crates/sqlbuild-core/src/test_command/_helpers/coded_error.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs @@ -2,9 +2,10 @@ use crate::cli_text::models::CliStyle; -/// `[CODE] message`, later message lines muted, then the help line when there is one. +/// `error[CODE]: message` or `[CODE] message`, later lines muted, then any help line. pub(crate) fn coded_error( (code, message, help): (&str, &str, Option<&str>), + include_error_label: bool, style: CliStyle, ) -> String { let mut lines = message.split('\n'); @@ -17,7 +18,11 @@ pub(crate) fn coded_error( }; let mut rendered: String = format!( "{} {rendered_message}", - style.error_strong(&format!("[{code}]")) + style.error_strong(&if include_error_label { + format!("error[{code}]:") + } else { + format!("[{code}]") + }) ); if let Some(help) = help { rendered = format!("{rendered}\n {} {help}", style.muted("= help:")); diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs index f7c744da86..34f8daf1d2 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod bounded_sections; +pub(crate) mod coded_error; pub(crate) mod dbt_plan; pub(crate) mod dbt_plan_json; pub(crate) mod diagnostics; diff --git a/crates/sqlbuild-core/src/cli_text/main/format_coded_error.rs b/crates/sqlbuild-core/src/cli_text/main/format_coded_error.rs new file mode 100644 index 0000000000..092b0b4b39 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_coded_error.rs @@ -0,0 +1,14 @@ +//! A result-stored coded error, as CLI expected errors render it. + +use crate::cli_text::_helpers::coded_error::coded_error; +use crate::cli_text::models::CliStyle; + +/// `format_coded_error`: the coded message, later lines muted, then any help line. +#[must_use] +pub fn format_coded_error( + (code, message, help): (&str, &str, Option<&str>), + include_error_label: bool, + style: CliStyle, +) -> String { + coded_error((code, message, help), include_error_label, style) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_tree_connector.rs b/crates/sqlbuild-core/src/cli_text/main/format_tree_connector.rs new file mode 100644 index 0000000000..423012e01f --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_tree_connector.rs @@ -0,0 +1,10 @@ +//! The dim connector of one tree row. + +use crate::cli_text::_helpers::layout::tree_connector; +use crate::cli_text::models::CliStyle; + +/// `tree_connector`: a dim branch, or the last branch of a group. +#[must_use] +pub fn format_tree_connector(style: CliStyle, last: bool) -> String { + tree_connector(style, last) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/mod.rs b/crates/sqlbuild-core/src/cli_text/main/mod.rs index 284e6fec08..8208efb9bd 100644 --- a/crates/sqlbuild-core/src/cli_text/main/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/main/mod.rs @@ -1,5 +1,6 @@ pub mod dbt_interop_plan_warnings; pub mod format_aligned_name_value; +pub mod format_coded_error; pub mod format_compile_text; pub mod format_completion_line; pub mod format_count_noun; @@ -12,6 +13,7 @@ pub mod format_plan_text; pub mod format_status_cell; pub mod format_summary_footer; pub mod format_surface_header; +pub mod format_tree_connector; pub mod resolve_name_column_width; pub mod semantic_coverage_notice; pub mod split_count_header; diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index f5423d1dbb..6b08424e07 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -18,6 +18,7 @@ pub mod panics; pub mod posix_path; pub mod progress; pub mod redaction; +pub mod seed_command; pub mod terminal_events; pub mod test_command; pub mod text; diff --git a/crates/sqlbuild-core/src/seed_command/constants.rs b/crates/sqlbuild-core/src/seed_command/constants.rs new file mode 100644 index 0000000000..ea0d36d5b3 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/constants.rs @@ -0,0 +1,6 @@ +//! Seed output labels and widths. + +pub(crate) const OK_STATUS: &str = "OK"; +pub(crate) const FAIL_STATUS: &str = "FAIL"; +pub(crate) const SEED_NAME_WIDTH: usize = 30; +pub(crate) const STATUS_WIDTH: usize = 6; diff --git a/crates/sqlbuild-core/src/seed_command/main/mod.rs b/crates/sqlbuild-core/src/seed_command/main/mod.rs new file mode 100644 index 0000000000..b217f85970 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/main/mod.rs @@ -0,0 +1,4 @@ +pub mod render_seed_completion; +pub mod render_seed_ready; +pub mod render_seed_row; +pub mod seed_exit_code; diff --git a/crates/sqlbuild-core/src/seed_command/main/render_seed_completion.rs b/crates/sqlbuild-core/src/seed_command/main/render_seed_completion.rs new file mode 100644 index 0000000000..501c94a016 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/main/render_seed_completion.rs @@ -0,0 +1,41 @@ +//! The seed completion line. + +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, CompletionState, SummaryCount}; + +fn count(label: &str, value: usize) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value: i64::try_from(value).unwrap_or(i64::MAX), + } +} + +/// A blank line and the completion state with PASS, WARN, FAIL, SKIP and TOTAL counts. +#[must_use] +pub fn render_seed_completion( + (passed, failed, total): (usize, usize, usize), + elapsed_seconds: f64, + style: CliStyle, +) -> String { + let summary: String = format_summary_footer( + style, + &[ + count("PASS", passed), + count("WARN", 0), + count("FAIL", failed), + count("SKIP", 0), + count("TOTAL", total), + ], + Some(&format!("{elapsed_seconds:.2}s")), + ); + let (state, label) = if failed == 0 { + (CompletionState::Ok, "Completed successfully") + } else { + (CompletionState::Fail, "Completed with errors") + }; + format!( + "\n{}\n", + format_completion_line(style, state, label, Some(&summary)) + ) +} diff --git a/crates/sqlbuild-core/src/seed_command/main/render_seed_ready.rs b/crates/sqlbuild-core/src/seed_command/main/render_seed_ready.rs new file mode 100644 index 0000000000..70e258ed4d --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/main/render_seed_ready.rs @@ -0,0 +1,24 @@ +//! The header announcing the selected seeds. + +use crate::cli_text::main::format_surface_header::format_surface_header; +use crate::cli_text::main::format_tree_connector::format_tree_connector; +use crate::cli_text::models::CliStyle; + +/// `Seed ready selected`, then the `Seeds ()` tree and a blank line. +#[must_use] +pub fn render_seed_ready(names: &[String], style: CliStyle) -> String { + let count: usize = names.len(); + let mut text: String = format!( + "\n{}\n\n{}\n", + format_surface_header(style, "Seed ready", &format!("{count} selected")), + style.section(&format!("Seeds ({count})")) + ); + for (index, name) in names.iter().enumerate() { + text.push_str(&format!( + "{} {name}\n", + format_tree_connector(style, index + 1 == count) + )); + } + text.push('\n'); + text +} diff --git a/crates/sqlbuild-core/src/seed_command/main/render_seed_row.rs b/crates/sqlbuild-core/src/seed_command/main/render_seed_row.rs new file mode 100644 index 0000000000..b6b9337b4b --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/main/render_seed_row.rs @@ -0,0 +1,34 @@ +//! The progress row of one finished seed. + +use crate::cli_text::main::format_coded_error::format_coded_error; +use crate::cli_text::models::CliStyle; +use crate::seed_command::constants::{FAIL_STATUS, OK_STATUS, SEED_NAME_WIDTH, STATUS_WIDTH}; +use crate::seed_command::models::SeedRow; + +/// ` / seed `, then its error line when there is one. +#[must_use] +pub fn render_seed_row(row: &SeedRow, style: CliStyle) -> String { + let status: String = style.status(if row.succeeded { + OK_STATUS + } else { + FAIL_STATUS + }); + let duration: String = row + .duration_ms + .map(|milliseconds| format!("{:.2}s", milliseconds / 1000.0)) + .unwrap_or_default(); + let mut text: String = format!( + " {}/{} seed {: { + format_coded_error((code, message, row.error_help.as_deref()), true, style) + } + None => message.clone(), + }; + text.push_str(&format!(" {rendered}\n")); + } + text +} diff --git a/crates/sqlbuild-core/src/seed_command/main/seed_exit_code.rs b/crates/sqlbuild-core/src/seed_command/main/seed_exit_code.rs new file mode 100644 index 0000000000..66d0312466 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/main/seed_exit_code.rs @@ -0,0 +1,7 @@ +//! The exit code of `sqb seed`. + +/// 1 when any seed failed, else 0. +#[must_use] +pub fn seed_exit_code(failed: usize) -> i32 { + i32::from(failed > 0) +} diff --git a/crates/sqlbuild-core/src/seed_command/mod.rs b/crates/sqlbuild-core/src/seed_command/mod.rs new file mode 100644 index 0000000000..b87e640489 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/mod.rs @@ -0,0 +1,8 @@ +//! `sqb seed`: the ready header, per-seed rows, the completion summary and the exit code. + +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/seed_command/models.rs b/crates/sqlbuild-core/src/seed_command/models.rs new file mode 100644 index 0000000000..f28258b8f9 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/models.rs @@ -0,0 +1,14 @@ +//! One finished seed as its progress row shows it. + +/// A completed seed load. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct SeedRow { + pub name: String, + pub ordinal: usize, + pub total: usize, + pub succeeded: bool, + pub duration_ms: Option, + pub error_message: Option, + pub error_code: Option, + pub error_help: Option, +} diff --git a/crates/sqlbuild-core/src/seed_command/tests.rs b/crates/sqlbuild-core/src/seed_command/tests.rs new file mode 100644 index 0000000000..c08eaa61e2 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_seed_command.rs"] +mod seed_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/seed_command/tests/test_seed_command.rs b/crates/sqlbuild-core/src/seed_command/tests/test_seed_command.rs new file mode 100644 index 0000000000..b9b538f11f --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/tests/test_seed_command.rs @@ -0,0 +1,119 @@ +use crate::cli_text::models::CliStyle; +use crate::seed_command::main::render_seed_completion::render_seed_completion; +use crate::seed_command::main::render_seed_ready::render_seed_ready; +use crate::seed_command::main::render_seed_row::render_seed_row; +use crate::seed_command::main::seed_exit_code::seed_exit_code; +use crate::seed_command::models::SeedRow; +use crate::seed_command::tests::test_types::{CompletionTestCase, ReadyTestCase, RowTestCase}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_selected_seeds_when_rendering_ready_then_lists_them_as_a_tree() { + let test_cases = [ + ReadyTestCase { + description: "two seeds end with the last branch", + names: vec!["first_seed".to_owned(), "second_seed".to_owned()], + expected_text: "\nSeed ready 2 selected\n\nSeeds (2)\n├── first_seed\n└── second_seed\n\n", + }, + ReadyTestCase { + description: "no seeds keep the header", + names: Vec::new(), + expected_text: "\nSeed ready 0 selected\n\nSeeds (0)\n\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_seed_ready(&test_case.names, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_finished_seed_when_rendering_row_then_pads_name_and_shows_error() { + let test_cases = [ + RowTestCase { + description: "a success shows its duration", + row: SeedRow { + name: "first_seed".to_owned(), + ordinal: 1, + total: 2, + succeeded: true, + duration_ms: Some(1234.0), + ..SeedRow::default() + }, + expected_text: " 1/2 seed first_seed OK 1.23s\n", + }, + RowTestCase { + description: "a coded failure renders the labelled error and help", + row: SeedRow { + name: "bad_seed".to_owned(), + ordinal: 2, + total: 2, + error_message: Some("bad value\ndetail".to_owned()), + error_code: Some("S001".to_owned()), + error_help: Some("fix the CSV".to_owned()), + ..SeedRow::default() + }, + expected_text: " 2/2 seed bad_seed FAIL \n error[S001]: bad value\ndetail\n = help: fix the CSV\n", + }, + RowTestCase { + description: "an uncoded failure shows the message", + row: SeedRow { + name: "bad_seed".to_owned(), + ordinal: 1, + total: 1, + duration_ms: Some(5.0), + error_message: Some("boom".to_owned()), + ..SeedRow::default() + }, + expected_text: " 1/1 seed bad_seed FAIL 0.01s\n boom\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_seed_row(&test_case.row, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_completing_then_summarizes_and_resolves_exit_code() { + let test_cases = [ + CompletionTestCase { + description: "all seeds loaded", + counts: (2, 0, 2), + expected_text: "\n✓ Completed successfully PASS=2 WARN=0 FAIL=0 SKIP=0 TOTAL=2 (1.50s)\n", + expected_exit_code: 0, + }, + CompletionTestCase { + description: "a failed seed fails the run", + counts: (1, 1, 2), + expected_text: "\n✗ Completed with errors PASS=1 WARN=0 FAIL=1 SKIP=0 TOTAL=2 (1.50s)\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_seed_completion(test_case.counts, 1.5, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + assert_eq!( + seed_exit_code(test_case.counts.1), + test_case.expected_exit_code, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/seed_command/tests/test_types.rs b/crates/sqlbuild-core/src/seed_command/tests/test_types.rs new file mode 100644 index 0000000000..278331c6b7 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/tests/test_types.rs @@ -0,0 +1,20 @@ +use crate::seed_command::models::SeedRow; + +pub(super) struct ReadyTestCase { + pub(super) description: &'static str, + pub(super) names: Vec, + pub(super) expected_text: &'static str, +} + +pub(super) struct RowTestCase { + pub(super) description: &'static str, + pub(super) row: SeedRow, + pub(super) expected_text: &'static str, +} + +pub(super) struct CompletionTestCase { + pub(super) description: &'static str, + pub(super) counts: (usize, usize, usize), + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} diff --git a/crates/sqlbuild-core/src/test_command/_helpers/mod.rs b/crates/sqlbuild-core/src/test_command/_helpers/mod.rs deleted file mode 100644 index bb61fa37f5..0000000000 --- a/crates/sqlbuild-core/src/test_command/_helpers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod coded_error; diff --git a/crates/sqlbuild-core/src/test_command/main/render_test_completion.rs b/crates/sqlbuild-core/src/test_command/main/render_test_completion.rs index 8718d229b8..fcf1d9decd 100644 --- a/crates/sqlbuild-core/src/test_command/main/render_test_completion.rs +++ b/crates/sqlbuild-core/src/test_command/main/render_test_completion.rs @@ -1,8 +1,8 @@ //! The test summary footer and failure details. +use crate::cli_text::main::format_coded_error::format_coded_error; use crate::cli_text::main::format_summary_footer::format_summary_footer; use crate::cli_text::models::{CliStyle, SummaryCount}; -use crate::test_command::_helpers::coded_error::coded_error; use crate::test_command::constants::{DEFAULT_ERROR, ERROR_LABEL, ERROR_LABEL_WIDTH}; use crate::test_command::models::TestFailure; use crate::text::main::python_splitlines::python_splitlines; @@ -21,7 +21,9 @@ fn failure_lines(failure: &TestFailure, style: CliStyle) -> Vec { .filter(|text| !text.is_empty()) .unwrap_or(DEFAULT_ERROR); let rendered: String = match &failure.error_code { - Some(code) => coded_error((code, message, failure.error_help.as_deref()), style), + Some(code) => { + format_coded_error((code, message, failure.error_help.as_deref()), false, style) + } None => message.to_owned(), }; let mut split: Vec<&str> = python_splitlines(&rendered); diff --git a/crates/sqlbuild-core/src/test_command/mod.rs b/crates/sqlbuild-core/src/test_command/mod.rs index 34acbbf1d3..8c866e79cf 100644 --- a/crates/sqlbuild-core/src/test_command/mod.rs +++ b/crates/sqlbuild-core/src/test_command/mod.rs @@ -1,6 +1,5 @@ //! `sqb test`: test labels, plan inspection, completion summary and exit code. -pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 850e9ec734..179b9abaf4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -5,7 +5,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; use crate::bindings::_helpers::{ check, clone, command_runtime, cost_report, dbt_interop, debug_command, diff_report, freshness, - janitor, load, presentation_text, result_output, sql_test_command, workspace_commands, + janitor, load, presentation_text, result_output, seed_command, sql_test_command, + workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -28,5 +29,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { workspace_commands::registration::register(module)?; check::registration::register(module)?; sql_test_command::registration::register(module)?; + seed_command::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 5d348fc503..1ab5b302a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -44,6 +44,7 @@ pub(crate) mod render; pub(crate) mod result_output; pub(crate) mod rules; pub(crate) mod scopes; +pub(crate) mod seed_command; pub(crate) mod semantic_checks; pub(crate) mod semantic_shapes; pub(crate) mod sql_test_command; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs new file mode 100644 index 0000000000..1b087534ea --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -0,0 +1,493 @@ +//! `sqb seed`: compile the selected seeds, load them, and report. + +use std::collections::HashMap; +use std::time::Instant; + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::seed_command::main::render_seed_completion::render_seed_completion; +use sqlbuild_core::seed_command::main::render_seed_ready::render_seed_ready; +use sqlbuild_core::seed_command::main::render_seed_row::render_seed_row; +use sqlbuild_core::seed_command::main::seed_exit_code::seed_exit_code; +use sqlbuild_core::seed_command::models::SeedRow; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main.static_command"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const SEED_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed"; +const SEED_STATE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed_state"; +const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; +const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; +const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; +const SUCCESS_STATUS: &str = "success"; +const FAILED_STATUS: &str = "failed"; +const EXECUTION_COMMAND: &str = "sqb seed"; + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn status_value(result: &Bound<'_, PyAny>) -> PyResult { + result.getattr("status")?.getattr("value")?.extract() +} + +/// Forward positional connection timing to a reporter that takes keywords. +#[pyclass(module = "sqlbuild._native", frozen)] +struct SeedConnectionRelay { + reporter: Py, +} + +#[pymethods] +impl SeedConnectionRelay { + fn on_connection_complete( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.forward( + py, + "on_connection_complete", + (connection_count, elapsed_seconds), + ) + } + + fn on_connection_error( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.forward( + py, + "on_connection_error", + (connection_count, elapsed_seconds), + ) + } +} + +impl SeedConnectionRelay { + fn forward( + &self, + py: Python<'_>, + method: &str, + (connection_count, elapsed_seconds): (i64, f64), + ) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("connection_count", connection_count)?; + options.set_item("elapsed_seconds", elapsed_seconds)?; + self.reporter + .bind(py) + .call_method(method, (), Some(&options))?; + Ok(()) + } +} + +fn connection_relay<'py>( + reporter: &Bound<'py, PyAny>, +) -> PyResult> { + Bound::new( + reporter.py(), + SeedConnectionRelay { + reporter: reporter.clone().unbind(), + }, + ) +} + +/// Write each finished seed's row and its execution event. +#[pyclass(module = "sqlbuild._native", frozen)] +struct SeedCompletionRelay { + stream: Py, + use_color: bool, + ordinals: HashMap, + total: usize, + event_writer: Py, + plan: Py, + projector: Option>, +} + +#[pymethods] +impl SeedCompletionRelay { + fn on_seed_complete(&self, py: Python<'_>, result: Bound<'_, PyAny>) -> PyResult<()> { + let name: String = result.getattr("seed_name")?.extract()?; + let mut result: Bound<'_, PyAny> = result; + if let Some(projector) = &self.projector { + let duration: Bound<'_, PyAny> = call_with( + &projector.bind(py).getattr("consume_resource_terminal")?, + &[ + ("resource_name", text(py, &name)), + ("resource_id", text(py, &format!("seed:{name}"))), + ], + )?; + if !duration.is_none() { + let milliseconds: Bound<'_, PyAny> = + py.import("builtins")?.getattr("int")?.call1((duration,))?; + let changes: Bound<'_, PyDict> = PyDict::new(py); + changes.set_item("duration_ms", milliseconds)?; + result = py + .import("dataclasses")? + .getattr("replace")? + .call((result,), Some(&changes))?; + } + } + let ordinal: usize = *self + .ordinals + .get(&name) + .ok_or_else(|| PyKeyError::new_err(name.clone()))?; + let style: CliStyle = CliStyle { + use_color: self.use_color, + }; + let row: SeedRow = SeedRow { + name, + ordinal, + total: self.total, + succeeded: status_value(&result)? == SUCCESS_STATUS, + duration_ms: result.getattr("duration_ms")?.extract()?, + error_message: result.getattr("error_message")?.extract()?, + error_code: result.getattr("error_code")?.extract()?, + error_help: result.getattr("error_help")?.extract()?, + }; + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + stream.call_method1("write", (render_seed_row(&row, style),))?; + stream.call_method0("flush")?; + call_with( + &self.event_writer.bind(py).getattr("write_build_result")?, + &[ + ("result", result), + ("plan", self.plan.bind(py).clone()), + ("command", text(py, "seed")), + ], + )?; + Ok(()) + } +} + +/// Compile the selected seeds, reading their stored fingerprints when any are selected. +fn compile_seeds<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; + let adapter: Bound<'py, PyAny> = invocation.getattr("adapter")?; + let connection_config: Bound<'py, PyAny> = invocation.getattr("connection_config")?; + let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; + let connection_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", invocation.getattr("adapter_name")?), + ("stream", stream.clone()), + ("use_color", invocation.getattr("use_color")?), + ], + )?; + stream.call_method1("write", ("\n",))?; + stream.call_method0("flush")?; + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.getattr("effective_project_dir")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let options: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &[ + ("selected_target", request.getattr("selected_target")?), + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("connection_config", connection_config.clone()), + ("cli_vars", request.getattr("cli_vars")?), + ("external_sql_reference_resolver", resolver), + ], + )?; + let context: Bound<'py, PyAny> = call_with( + &imported(py, STATIC_COMMAND_MODULE, "compile_static_command_context")?, + &[ + ("discovered_inputs", discovered), + ("adapter", adapter.clone()), + ("options", options), + ], + )?; + let project: Bound<'py, PyAny> = context.getattr("project")?; + let scope: Bound<'py, PyAny> = context.getattr("scope")?; + let selected_keys: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + let mut selected: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed: Bound<'py, PyAny> = seed?; + if selected_keys.contains(seed.getattr("key")?)? { + selected.push(seed); + } + } + let fingerprints: Bound<'py, PyAny> = if selected.is_empty() { + PyDict::new(py).into_any() + } else { + let relay: Bound<'py, SeedConnectionRelay> = connection_relay(&connection_progress)?; + let hooks: Bound<'py, PyAny> = call_with( + &imported(py, RUNTIME_CONTRACT_MODELS, "ConnectionHooks")?, + &[ + ( + "on_connection_start", + connection_progress.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + relay.getattr("on_connection_complete")?, + ), + ("on_connection_error", relay.getattr("on_connection_error")?), + ], + )?; + let connection: Bound<'py, PyAny> = call_with( + &imported(py, OPEN_CONNECTION_MODULE, "open_connection_with_hooks")?, + &[ + ("adapter", adapter.clone()), + ("connection_config", connection_config), + ("hooks", hooks), + ], + )?; + let read: PyResult> = call_with( + &imported(py, SEED_STATE_MODULE, "read_selected_seed_fingerprints")?, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("seeds", PyTuple::new(py, &selected)?.into_any()), + ], + ); + let closed: PyResult> = adapter.call_method1("close", (connection,)); + let read: Bound<'py, PyAny> = read?; + closed?; + read + }; + let plan_output: Bound<'py, PyAny> = call_with( + &imported(py, SEED_PLAN_MODULE, "build_seed_command_plan")?, + &[ + ("project", project), + ("scope", scope), + ("relations", context.getattr("relations")?), + ("fingerprints", fingerprints), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "build_static_pipeline_result")?, + &[("context", context), ("plan_output", plan_output)], + ) +} + +fn seed_names(pipeline: &Bound<'_, PyAny>) -> PyResult> { + pipeline + .getattr("plan_output")? + .getattr("seed_entries")? + .try_iter()? + .map(|entry| entry?.getattr("name")?.extract()) + .collect() +} + +/// Write the ready header, the selected seeds and the execution header. +fn write_header( + invocation: &Bound<'_, PyAny>, + pipeline: &Bound<'_, PyAny>, + concurrency: i64, +) -> PyResult<()> { + let py: Python<'_> = invocation.py(); + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + let use_color: Bound<'_, PyAny> = invocation.getattr("use_color")?; + let style: CliStyle = CliStyle { + use_color: use_color.is_truthy()?, + }; + stream.call_method1("write", (render_seed_ready(&seed_names(pipeline)?, style),))?; + let header: Bound<'_, PyDict> = PyDict::new(py); + header.set_item("stream", &stream)?; + header.set_item("command", EXECUTION_COMMAND)?; + header.set_item("target", py.None())?; + header.set_item("concurrency", concurrency)?; + header.set_item("use_color", use_color)?; + imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&header))?; + Ok(()) +} + +/// Load the planned seeds, writing one row per finished seed. +fn load_seeds<'py>( + invocation: &Bound<'py, PyAny>, + pipeline: &Bound<'py, PyAny>, + concurrency: i64, +) -> PyResult>> { + let py: Python<'py> = invocation.py(); + let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; + let use_color: Bound<'py, PyAny> = invocation.getattr("use_color")?; + let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let project: Bound<'py, PyAny> = pipeline.getattr("project")?; + let names: Vec = seed_names(pipeline)?; + let ordinals: HashMap = names + .iter() + .enumerate() + .map(|(index, name)| (name.clone(), index + 1)) + .collect(); + let event_writer: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; + let projector: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + let projector: Option> = if projector.is_none() { + None + } else { + let order: Bound<'py, PyDict> = PyDict::new(py); + for (index, name) in names.iter().enumerate() { + order.set_item(name, index + 1)?; + } + call_with( + &projector.getattr("configure_resources")?, + &[ + ("ordinals", order.into_any()), + ("total", PyInt::new(py, names.len()).into_any()), + ], + )?; + Some(projector.unbind()) + }; + let completion: Bound<'py, SeedCompletionRelay> = Bound::new( + py, + SeedCompletionRelay { + stream: stream.clone().unbind(), + use_color: use_color.is_truthy()?, + ordinals, + total: names.len(), + event_writer: event_writer.clone().unbind(), + plan: plan.clone().unbind(), + projector, + }, + )?; + let connection_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", invocation.getattr("adapter_name")?), + ("stream", stream), + ( + "blank_line_after_complete", + PyBool::new(py, true).to_owned().into_any(), + ), + ("use_color", use_color), + ], + )?; + let relay: Bound<'py, SeedConnectionRelay> = connection_relay(&connection_progress)?; + let loaded: PyResult>> = (|| { + call_with( + &imported(py, EXECUTOR_RUN_MODULE, "run_seed_pipeline")?, + &[ + ("plan", plan), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("adapter", invocation.getattr("adapter")?), + ("max_concurrency", PyInt::new(py, concurrency).into_any()), + ("run_id", project.getattr("run_id")?), + ( + "query_change_tracking", + project + .getattr("settings")? + .getattr("query_change_tracking")?, + ), + ("on_seed_complete", completion.getattr("on_seed_complete")?), + ( + "on_connection_start", + connection_progress.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + relay.getattr("on_connection_complete")?, + ), + ("on_connection_error", relay.getattr("on_connection_error")?), + ], + )? + .try_iter()? + .collect() + })(); + let closed: PyResult> = event_writer.call_method0("close"); + let results: Vec> = loaded?; + closed?; + Ok(results) +} + +/// Run one `sqb seed` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_seed_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let invocation: Bound<'_, PyAny> = call_with( + &imported(py, INVOCATION_MODULE, "resolve_command_invocation")?, + &[ + ("request", request.clone()), + ( + "invocation_type", + imported(py, COMMAND_MODELS, "SeedInvocation")?, + ), + ], + )?; + let pipeline: Bound<'_, PyAny> = compile_seeds(request, &invocation)?; + let requested: Bound<'_, PyAny> = request.getattr("concurrency")?; + let concurrency: i64 = if requested.is_none() { + pipeline + .getattr("project")? + .getattr("settings")? + .getattr("concurrency")? + .extract::()? + } else { + requested.extract::()? + } + .max(1); + write_header(&invocation, &pipeline, concurrency)?; + let started: Instant = Instant::now(); + let results: Vec> = load_seeds(&invocation, &pipeline, concurrency)?; + let elapsed: f64 = started.elapsed().as_secs_f64(); + let mut passed: usize = 0; + let mut failed: usize = 0; + for result in &results { + match status_value(result)?.as_str() { + SUCCESS_STATUS => passed += 1, + FAILED_STATUS => failed += 1, + _ => {} + } + } + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + let style: CliStyle = CliStyle { + use_color: invocation.getattr("use_color")?.is_truthy()?, + }; + stream.call_method1( + "write", + (render_seed_completion( + (passed, failed, results.len()), + elapsed, + style, + ),), + )?; + stream.call_method0("flush")?; + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_seed_execution_json")?, + &[ + ("results", PyTuple::new(py, &results)?.into_any()), + ("plan", pipeline.getattr("plan_output")?), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(seed_exit_code(failed)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_seed_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/mod.rs new file mode 100644 index 0000000000..2ef3aae6d5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod command; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/registration.rs new file mode 100644 index 0000000000..7fcfbefc33 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/registration.rs @@ -0,0 +1,9 @@ +//! Register the native `sqb seed` command. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::seed_command::command; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + command::register(module) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 85d17eeee3..6e0b59ac04 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -30,6 +30,7 @@ from sqlbuild.cli.commands.models import ( QueryDiffPreparation, RefactorCommandRequest, ScopeCommandRequest, + SeedCommandRequest, TestCommandRequest, ) from sqlbuild.cli.commands.types import CompileLineageMode, ScopeIndexLoader @@ -2875,6 +2876,7 @@ def write_check_results( ) -> None: ... def format_check_json(*, results: tuple[PythonCheckExecutionResult, ...]) -> str: ... def run_test_command(*, request: TestCommandRequest) -> int: ... +def run_seed_command(*, request: SeedCommandRequest) -> int: ... def format_parameterized_test_label( *, name: str, diff --git a/src/sqlbuild/cli/commands/_helpers/seed/execution.py b/src/sqlbuild/cli/commands/_helpers/seed/execution.py deleted file mode 100644 index 16b4993020..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/seed/execution.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Seed command execution phases.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from dataclasses import replace -from typing import TextIO - -from sqlbuild._native import ( - ConnectionProgressReporter, - ExecutionEventWriter, - NativeProgressProjector, - current_native_progress_projector, -) -from sqlbuild.cli.commands.models import ( - SeedExecutionPreparation, - SeedInvocation, - SeedRunOutcome, -) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.build.models import SeedExecutionResult -from sqlbuild.executor.build.types import ExecutionStatus -from sqlbuild.executor.pipeline.main.run import run_seed_pipeline -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.coded_error_text import format_coded_error - - -def execute_seed_plan( - *, invocation: SeedInvocation, preparation: SeedExecutionPreparation -) -> SeedRunOutcome: - """Execute compiled seed entries.""" - - start: float = time.monotonic() - event_writer: ExecutionEventWriter = ExecutionEventWriter() - on_complete: Callable[[SeedExecutionResult], None] = _build_on_complete( - stream=invocation.progress_stream, - use_color=invocation.use_color, - seed_order={ - seed_entry.name: index - for index, seed_entry in enumerate( - preparation.pipeline_result.plan_output.seed_entries, start=1 - ) - }, - total_count=len(preparation.pipeline_result.plan_output.seed_entries), - event_writer=event_writer, - plan=preparation.pipeline_result.plan_output, - ) - execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=invocation.adapter_name, - stream=invocation.progress_stream, - blank_line_after_complete=True, - use_color=invocation.use_color, - ) - try: - results: tuple[SeedExecutionResult, ...] = run_seed_pipeline( - plan=preparation.pipeline_result.plan_output, - connection_config=invocation.connection_config, - adapter=invocation.adapter, - max_concurrency=preparation.effective_concurrency, - run_id=preparation.pipeline_result.project.run_id, - query_change_tracking=preparation.pipeline_result.project.settings.query_change_tracking, - on_seed_complete=on_complete, - on_connection_start=execution_connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ) - finally: - event_writer.close() - return SeedRunOutcome(results=results, elapsed=time.monotonic() - start) - - -def _build_on_complete( - *, - stream: TextIO, - use_color: bool, - seed_order: dict[str, int], - total_count: int, - event_writer: ExecutionEventWriter, - plan: PlanOutput, -) -> Callable[[SeedExecutionResult], None]: - projector: NativeProgressProjector | None = current_native_progress_projector() - if projector is not None: - projector.configure_resources(ordinals=seed_order, total=total_count) - - def _on_complete(result: SeedExecutionResult) -> None: - if projector is not None: - canonical_duration_ms: float | None = projector.consume_resource_terminal( - resource_name=result.seed_name, resource_id=f"seed:{result.seed_name}" - ) - if canonical_duration_ms is not None: - result = replace(result, duration_ms=int(canonical_duration_ms)) - status_text: str = "OK" if result.status == ExecutionStatus.SUCCESS else "FAIL" - style: CliStyle = CliStyle(use_color=use_color) - status: str = style.status(status=status_text) - duration: str = "" - if result.duration_ms is not None: - seconds: float = result.duration_ms / 1000.0 - duration = f"{seconds:.2f}s" - ordinal: int = seed_order[result.seed_name] - stream.write( - f" {ordinal}/{total_count} seed {result.seed_name:<30} {status:<6} {duration}\n" - ) - if result.error_message is not None: - stream.write(f" {_format_seed_error(result=result, use_color=use_color)}\n") - stream.flush() - event_writer.write_build_result(result=result, plan=plan, command="seed") - - return _on_complete - - -def _format_seed_error(*, result: SeedExecutionResult, use_color: bool) -> str: - if result.error_message is None: - return "" - if result.error_code is None: - return result.error_message - return format_coded_error( - code=result.error_code, - message=result.error_message, - help=result.error_help, - use_color=use_color, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/seed/invocation.py b/src/sqlbuild/cli/commands/_helpers/seed/invocation.py deleted file mode 100644 index 3fdb7028db..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/seed/invocation.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Seed command invocation resolution phase.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( - resolve_command_invocation, -) -from sqlbuild.cli.commands.models import SeedCommandRequest, SeedInvocation - - -def resolve_seed_invocation(*, request: SeedCommandRequest) -> SeedInvocation: - """Resolve discovery, adapter, connection, and output context for seed.""" - - return resolve_command_invocation(request=request, invocation_type=SeedInvocation) diff --git a/src/sqlbuild/cli/commands/_helpers/seed/outputs.py b/src/sqlbuild/cli/commands/_helpers/seed/outputs.py deleted file mode 100644 index 569630a729..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/seed/outputs.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Seed command output writing and exit-code phases.""" - -from __future__ import annotations - -from sqlbuild._native import ( - format_completion_line, - format_seed_execution_json, - format_summary_footer, - write_execution_header, - write_execution_json_output, -) -from sqlbuild.cli.commands.models import ( - SeedCommandRequest, - SeedExecutionPreparation, - SeedInvocation, - SeedRunOutcome, -) -from sqlbuild.executor.build.models import SeedExecutionResult -from sqlbuild.executor.build.types import ExecutionStatus -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.surface_header import format_surface_header -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.types import CompletionState - - -def write_seed_execution_header( - *, invocation: SeedInvocation, preparation: SeedExecutionPreparation -) -> None: - """Write seed ready, selected seed names, and execution headers.""" - - seed_count: int = len(preparation.pipeline_result.plan_output.seed_entries) - style: CliStyle = CliStyle(use_color=invocation.use_color) - ready_header: str = format_surface_header( - style=style, title="Seed ready", context=f"{seed_count} selected" - ) - invocation.progress_stream.write(f"\n{ready_header}\n\n") - invocation.progress_stream.write(f"{style.section(f'Seeds ({seed_count})')}\n") - seed_index: int - for seed_index, seed_entry in enumerate(preparation.pipeline_result.plan_output.seed_entries): - connector: str = tree_connector(style=style, last=seed_index == seed_count - 1) - invocation.progress_stream.write(f"{connector} {seed_entry.name}\n") - invocation.progress_stream.write("\n") - write_execution_header( - stream=invocation.progress_stream, - command="sqb seed", - target=None, - concurrency=preparation.effective_concurrency, - use_color=invocation.use_color, - ) - - -def write_seed_completion_output( - *, - request: SeedCommandRequest, - invocation: SeedInvocation, - preparation: SeedExecutionPreparation, - outcome: SeedRunOutcome, -) -> None: - """Write seed summary and optional JSON output.""" - - success_count: int = _success_count(outcome.results) - fail_count: int = _fail_count(outcome.results) - counts_summary: str = format_summary_footer( - counts=( - ("PASS", success_count), - ("WARN", 0), - ("FAIL", fail_count), - ("SKIP", 0), - ("TOTAL", len(outcome.results)), - ), - use_color=invocation.use_color, - elapsed=f"{outcome.elapsed:.2f}s", - ) - completion_message: str = format_completion_line( - style=CliStyle(use_color=invocation.use_color), - state=CompletionState.OK if fail_count == 0 else CompletionState.FAIL, - label="Completed successfully" if fail_count == 0 else "Completed with errors", - summary=counts_summary, - ) - invocation.progress_stream.write(f"\n{completion_message}\n") - invocation.progress_stream.flush() - write_execution_json_output( - payload=format_seed_execution_json( - results=outcome.results, - plan=preparation.pipeline_result.plan_output, - ), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def resolve_seed_exit_code(results: tuple[SeedExecutionResult, ...]) -> int: - """Resolve seed exit code from failed results.""" - - return 0 if _fail_count(results) == 0 else 1 - - -def _success_count(results: tuple[SeedExecutionResult, ...]) -> int: - return sum(1 for result in results if result.status == ExecutionStatus.SUCCESS) - - -def _fail_count(results: tuple[SeedExecutionResult, ...]) -> int: - return sum(1 for result in results if result.status == ExecutionStatus.FAILED) diff --git a/src/sqlbuild/cli/commands/_helpers/seed/planning.py b/src/sqlbuild/cli/commands/_helpers/seed/planning.py deleted file mode 100644 index e91cd09bb6..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/seed/planning.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Seed command plan compilation phase.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import ConnectionProgressReporter, build_static_pipeline_result -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.models import ( - SeedCommandRequest, - SeedExecutionPreparation, - SeedInvocation, -) -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, - StaticCommandContext, -) -from sqlbuild.compiler.planner.main.commands.seed import build_seed_command_plan -from sqlbuild.compiler.planner.main.commands.seed_state import read_selected_seed_fingerprints -from sqlbuild.runtime.contracts.main.open_connection import open_connection_with_hooks -from sqlbuild.runtime.contracts.models import ConnectionHooks - - -def prepare_seed_execution( - *, request: SeedCommandRequest, invocation: SeedInvocation -) -> SeedExecutionPreparation: - """Compile seed plan and resolve effective concurrency.""" - - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=invocation.adapter_name, - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - invocation.progress_stream.write("\n") - invocation.progress_stream.flush() - options: CompilePipelineOptions = CompilePipelineOptions( - selected_target=request.selected_target, - select=request.select, - exclude=request.exclude, - connection_config=invocation.connection_config, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ) - context: StaticCommandContext = compile_static_command_context( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - options=options, - ) - selected_seeds: tuple[CompiledSeedView, ...] = tuple( - seed for seed in context.project.seeds if seed.key in context.scope.selected_keys - ) - hooks: ConnectionHooks = ConnectionHooks( - on_connection_start=connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ) - fingerprints: dict[str, Fingerprint] = {} - if selected_seeds: - connection: Any = open_connection_with_hooks( - adapter=invocation.adapter, - connection_config=invocation.connection_config, - hooks=hooks, - ) - try: - fingerprints = read_selected_seed_fingerprints( - adapter=invocation.adapter, - connection=connection, - seeds=selected_seeds, - ) - finally: - invocation.adapter.close(connection) - pipeline_result: CompilePipelineResult = build_static_pipeline_result( - context=context, - plan_output=build_seed_command_plan( - project=context.project, - scope=context.scope, - relations=context.relations, - fingerprints=fingerprints, - ), - ) - effective_concurrency: int = max( - 1, - request.concurrency - if request.concurrency is not None - else pipeline_result.project.settings.concurrency, - ) - return SeedExecutionPreparation( - pipeline_result=pipeline_result, - effective_concurrency=effective_concurrency, - ) diff --git a/src/sqlbuild/cli/commands/main/execution/_seed.py b/src/sqlbuild/cli/commands/main/execution/_seed.py deleted file mode 100644 index c7261e2a56..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_seed.py +++ /dev/null @@ -1,40 +0,0 @@ -"""CLI seed command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.seed.execution import execute_seed_plan -from sqlbuild.cli.commands._helpers.seed.invocation import resolve_seed_invocation -from sqlbuild.cli.commands._helpers.seed.outputs import ( - resolve_seed_exit_code, - write_seed_completion_output, - write_seed_execution_header, -) -from sqlbuild.cli.commands._helpers.seed.planning import prepare_seed_execution -from sqlbuild.cli.commands.models import ( - SeedCommandRequest, - SeedExecutionPreparation, - SeedInvocation, - SeedRunOutcome, -) - - -def run_seed(request: SeedCommandRequest) -> int: - """Execute the seed command.""" - - invocation: SeedInvocation = resolve_seed_invocation(request=request) - preparation: SeedExecutionPreparation = prepare_seed_execution( - request=request, - invocation=invocation, - ) - write_seed_execution_header(invocation=invocation, preparation=preparation) - outcome: SeedRunOutcome = execute_seed_plan( - invocation=invocation, - preparation=preparation, - ) - write_seed_completion_output( - request=request, - invocation=invocation, - preparation=preparation, - outcome=outcome, - ) - return resolve_seed_exit_code(outcome.results) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 8d66df4b7c..3429c98306 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -61,7 +61,7 @@ ) from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput from sqlbuild.compiler.refactoring.models import RefactorProject - from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult + from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.janitor.models import JanitorPlan from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.integrations.dbt.models import DbtInitRequest @@ -699,22 +699,6 @@ class SeedInvocation: progress_stream: TextIO -@dataclass(frozen=True) -class SeedExecutionPreparation: - """Prepared seed execution settings and compiled pipeline.""" - - pipeline_result: CompilePipelineResult - effective_concurrency: int - - -@dataclass(frozen=True) -class SeedRunOutcome: - """Seed execution results and elapsed time.""" - - results: tuple[SeedExecutionResult, ...] - elapsed: float - - @dataclass(frozen=True) class TestCommandRequest: """CLI inputs for one test command invocation.""" diff --git a/src/sqlbuild/compiler/planner/main/commands/seed.py b/src/sqlbuild/compiler/planner/main/commands/_seed.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/commands/seed.py rename to src/sqlbuild/compiler/planner/main/commands/_seed.py diff --git a/src/sqlbuild/compiler/planner/main/commands/seed_state.py b/src/sqlbuild/compiler/planner/main/commands/_seed_state.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/commands/seed_state.py rename to src/sqlbuild/compiler/planner/main/commands/_seed_state.py diff --git a/src/sqlbuild/presentation/main/surface_header.py b/src/sqlbuild/presentation/main/surface_header.py deleted file mode 100644 index 2764efdca4..0000000000 --- a/src/sqlbuild/presentation/main/surface_header.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public command surface header rendering entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.structure import format_surface_header as _format_surface_header -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_surface_header(*, style: CliStyle, title: str, context: str | None = None) -> str: - """Render a command surface header: accent title plus dim context.""" - - return _format_surface_header(style=style, title=title, context=context) diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py index 9f96cf02db..91cb48d23d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py @@ -11,9 +11,9 @@ from sqlbuild.cli.commands import models as command_models from sqlbuild.cli.commands._helpers.audit.invocation import resolve_audit_invocation from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( + resolve_command_invocation, resolve_reported_command_invocation, ) -from sqlbuild.cli.commands._helpers.seed.invocation import resolve_seed_invocation from tests.integration.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( CommandInvocationTestCase, ) @@ -62,7 +62,10 @@ ), CommandInvocationTestCase( description="seed text output omits shared progress reporters", - resolve=resolve_seed_invocation, + resolve=partial( + resolve_command_invocation, + invocation_type=command_models.SeedInvocation, + ), request_type=command_models.SeedCommandRequest, json_output=False, expected_invocation_type_name="SeedInvocation", diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 8d032760ec..c9bd338579 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -19,10 +19,10 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope +from sqlbuild.compiler.planner.main.commands._seed import build_seed_command_plan +from sqlbuild.compiler.planner.main.commands._seed_state import read_selected_seed_fingerprints from sqlbuild.compiler.planner.main.commands._sql_test import build_test_command_plan from sqlbuild.compiler.planner.main.commands.audit import build_audit_command_plan -from sqlbuild.compiler.planner.main.commands.seed import build_seed_command_plan -from sqlbuild.compiler.planner.main.commands.seed_state import read_selected_seed_fingerprints from sqlbuild.compiler.planner.models import ( PlannerRelationsContext, PlannerScope, From e38684dab0157f05a700a639a569fc5d84b34445 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:53:49 +0100 Subject: [PATCH 36/37] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 51cabb67a0..37c11ad33e 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 79857 +remaining_lines = 79435 [[stays]] path = "executor" From 2e9f092696e8869a814dfda01c5ac68717f26f79 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:32:57 +0100 Subject: [PATCH 37/37] fix(cli): render sqb check rows before encoding check metadata --- .../bindings/_helpers/check/check_results.rs | 14 ++++++-- .../commands/_helpers/check/_test_types.py | 7 ++++ .../commands/_helpers/check/test_outputs.py | 32 +++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs index cf21b50935..92955c0b94 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs @@ -40,7 +40,15 @@ fn metadata_json(metadata: &Bound<'_, PyAny>) -> PyResult { .ok_or_else(|| PyValueError::new_err("check metadata is not JSON serializable")) } -/// One executor `PythonCheckExecutionResult` as the command renders it. +/// One executor `PythonCheckExecutionResult` as the JSON output renders it, metadata included. +fn check_result_json_text(result: &Bound<'_, PyAny>) -> PyResult { + Ok(CheckResultText { + metadata: metadata_json(&result.getattr("metadata")?)?, + ..check_result_text(result)? + }) +} + +/// One executor `PythonCheckExecutionResult` as the rows and summary render it, without metadata. pub(crate) fn check_result_text(result: &Bound<'_, PyAny>) -> PyResult { let severity: String = result.getattr("severity")?.getattr("value")?.extract()?; let status: CheckStatus = if result.getattr("passed")?.is_truthy()? { @@ -56,7 +64,7 @@ pub(crate) fn check_result_text(result: &Bound<'_, PyAny>) -> PyResult]) -> PyResult { continue; } } - texts.push(check_result_text(result)?); + texts.push(check_result_json_text(result)?); } check_json(&texts).map_err(|error| PyValueError::new_err(format!("{error:?}"))) } diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/_test_types.py index 29bee8a035..cb04cad911 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/_test_types.py @@ -17,3 +17,10 @@ class CheckOutputProjectionCase: expected_first_duration: str expected_second_duration: str expected_generic_terminal_count: int + + +@dataclass(frozen=True) +class CheckMetadataRowsCase: + description: str + metadata: dict[str, object] + expected_row_fragment: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py index 5eec82056d..53818cdf5a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py @@ -3,12 +3,16 @@ from __future__ import annotations from contextvars import Token +from datetime import UTC, datetime from io import StringIO import pytest from sqlbuild._native import NativeProgressProjector, write_check_results +from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult +from sqlbuild.python_nodes.types import PythonCheckSeverity from tests.unit.src.sqlbuild.cli.commands._helpers.check._test_types import ( + CheckMetadataRowsCase, CheckOutputProjectionCase, ) from tests.unit.src.sqlbuild.cli.commands._helpers.check.helpers import ( @@ -90,3 +94,31 @@ def test_given_two_completed_checks_when_second_row_fails_then_close_renders_unc assert output.count(f"check second OK {test_case.expected_second_duration}") == ( test_case.expected_generic_terminal_count ) + + +@pytest.mark.parametrize( + "test_case", + ( + CheckMetadataRowsCase( + description="metadata that JSON cannot encode does not block human rows", + metadata={"checked_at": datetime(2026, 1, 2, tzinfo=UTC)}, + expected_row_fragment="first", + ), + ), + ids=lambda case: case.description, +) +def test_given_non_json_metadata_when_rows_render_then_rows_are_written( + test_case: CheckMetadataRowsCase, +) -> None: + stream: StringIO = StringIO() + result: PythonCheckExecutionResult = PythonCheckExecutionResult( + node_name="first", + passed=False, + severity=PythonCheckSeverity.ERROR, + metadata=test_case.metadata, + ) + + write_check_results(stream=stream, results=(result,), use_color=False) + + assert test_case.expected_row_fragment in stream.getvalue() + assert "FAIL" in stream.getvalue()