diff --git a/Cargo.lock b/Cargo.lock index af826cf0c7..44860e0dd9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1210,6 +1210,9 @@ dependencies = [ [[package]] name = "sqlbuild-core" version = "0.156.1" +dependencies = [ + "regex", +] [[package]] name = "sqlbuild-cursor" @@ -1274,6 +1277,7 @@ dependencies = [ "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", + "sqlbuild-state", ] [[package]] @@ -1344,6 +1348,15 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "sqlbuild-state" +version = "0.156.1" +dependencies = [ + "sha2", + "sqlbuild-core", + "sqlbuild-cursor", +] + [[package]] name = "sqlparser" version = "0.58.0" diff --git a/Cargo.toml b/Cargo.toml index ad79fd0a8e..1eed898f30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/sqlbuild-core", "crates/sqlbuild-cursor", + "crates/sqlbuild-state", "crates/sqlbuild-pysource", "crates/sqlbuild-cache", "crates/sqlbuild-sqltext", @@ -26,6 +27,7 @@ resolver = "2" order = [ "sqlbuild-core", "sqlbuild-cursor", + "sqlbuild-state", "sqlbuild-pysource", "sqlbuild-cache", "sqlbuild-sqltext", @@ -104,6 +106,7 @@ sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } +sqlbuild-state = { path = "crates/sqlbuild-state" } stacker = "0.1" sqlparser = { version = "0.58", features = ["visitor"] } syn = { version = "2", features = ["full", "visit", "extra-traits"] } diff --git a/crates/README.md b/crates/README.md index f957e37ee2..3bfdbb0b5f 100644 --- a/crates/README.md +++ b/crates/README.md @@ -8,6 +8,7 @@ that can be tested and benchmarked with `cargo test` alone. |---|---| | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | | `sqlbuild-cursor` | Microbatch cursor algebra: typed cursor values, grain flooring and stepping, half-open interval clamping, merging and batch splitting, timestamp text normalization and calendar durations, with Python `datetime` results and errors. Zones other than fixed offsets are resolved through a caller-supplied lookup. | +| `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence and age policy, byte-compatible with the rows Python wrote. | | `sqlbuild-pysource` | Static reading of project Python source with the ruff parser: macro module structure, literal SQL arguments, definition locations and string-literal edits. It never imports or runs user code; CPython stays the authority on whether a file is valid Python. | | `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. | @@ -31,7 +32,7 @@ Dependencies point one way, from the top of this graph to the bottom: sqlbuild-python -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-config - -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-cursor -> sqlbuild-core + -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core sqlbuild-cli -> sqlbuild-cache, sqlbuild-core ``` @@ -56,6 +57,21 @@ The same check fails when the workspace version, the `pyproject.toml` version an `fensu check` applies the Rust structure rules inside each crate: other crates use items through `main/` entry points, `models.rs`, `types.rs` and `constants.rs`, not through `_helpers/`. +## Python module paths + +`sqlbuild-python` and `sqlbuild-cli` import Python by module path strings. The repository test +`tests/unit/scripts/native_module_paths` (`scripts/native_module_paths`) keeps them honest: + +- every `"sqlbuild...."` literal must be a complete dotted path that imports as a module or + resolves as a module attribute; constants whose name contains `LOGGER` hold `logging` + logger names and are not imported; +- attributes are checked where a literal module, or a module constant defined in the same file, + is passed with a literal name to an import helper (`imported(py, MODULE, "name")`), to + `import(MODULE)?.getattr("name")`, or written as a `("sqlbuild.module", "name")` pair; +- module paths are never built at runtime: a `format!` or `concat!` template may not start with + `sqlbuild.` or with a `{..PACKAGE..}.`/`{..MODULE..}.` constant. Write the complete module + path as a constant instead. + ## Compiler The compiler runs discovery, rendering and analysis natively. Generic compiler stages do not diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/evidence.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/evidence.rs new file mode 100644 index 0000000000..5e8ca37cb2 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/evidence.rs @@ -0,0 +1,110 @@ +//! Bounded, difference-aware rendering of example values and keys. + +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; + +use crate::diff_report::_helpers::numbers::grouped; +use crate::diff_report::constants::MAX_RENDERED_KEY_LENGTH; +use crate::diff_report::models::{DiffExampleOptions, RenderedValue}; + +fn first_difference(left: &[char], right: &[char]) -> Option { + if let Some(index) = left.iter().zip(right).position(|(a, b)| a != b) { + return Some(index); + } + (left.len() != right.len()).then(|| left.len().min(right.len())) +} + +fn render_value( + text: &[char], + first_difference: Option, + options: DiffExampleOptions, +) -> RenderedValue { + let original_length: usize = text.len(); + let full: String = text.iter().collect(); + if options.suppress_values { + return RenderedValue { + text: None, + original_length, + truncated: false, + suppressed: true, + first_difference, + }; + } + let Some(limit) = options + .max_value_length + .filter(|limit| original_length > *limit) + else { + return RenderedValue { + text: Some(full), + original_length, + truncated: false, + suppressed: false, + first_difference, + }; + }; + let difference: usize = first_difference.unwrap_or(0); + let context_before: usize = (limit / 3).min(limit.saturating_sub(1)); + let start: usize = difference + .saturating_sub(context_before) + .min(original_length - limit); + let end: usize = start + limit; + let fragment: String = text[start..end].iter().collect(); + RenderedValue { + text: Some(format!( + "{}{fragment}{}", + if start > 0 { "…" } else { "" }, + if end < original_length { "…" } else { "" } + )), + original_length, + truncated: true, + suppressed: false, + first_difference, + } +} + +/// Two unequal values rendered around their first differing character. +pub(crate) fn render_example_pair( + left: &str, + right: &str, + options: DiffExampleOptions, +) -> (RenderedValue, RenderedValue) { + let left: Vec = left.chars().collect(); + let right: Vec = right.chars().collect(); + let difference: Option = first_difference(&left, &right); + ( + render_value(&left, difference, options), + render_value(&right, difference, options), + ) +} + +/// One bounded value for terminal output. +pub(crate) fn display_example_value(value: &RenderedValue) -> String { + if value.suppressed { + return format!( + "", + grouped(value.original_length as i64) + ); + } + let rendered: &str = value.text.as_deref().unwrap_or_default(); + if value.truncated { + return format!( + "{rendered} [truncated; {} chars]", + grouped(value.original_length as i64) + ); + } + rendered.to_owned() +} + +/// Ordinary keys complete; exceptionally long keys shortened and identified by digest. +pub(crate) fn render_key_value(text: &str) -> String { + let length: usize = text.chars().count(); + if length <= MAX_RENDERED_KEY_LENGTH { + return text.to_owned(); + } + let digest: String = sha256_hex(text.as_bytes()); + let prefix: String = text.chars().take(MAX_RENDERED_KEY_LENGTH).collect(); + format!( + "{prefix}… [truncated; {} chars; sha256:{}]", + grouped(length as i64), + &digest[..12] + ) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/findings.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/findings.rs new file mode 100644 index 0000000000..671dad9ffe --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/findings.rs @@ -0,0 +1,35 @@ +//! Whether a diff found differences, and the resulting outcome. + +use crate::diff_report::constants::{FINDINGS_OUTCOME, PASS_OUTCOME}; +use crate::diff_report::models::DiffModelFacts; + +pub(crate) fn has_schema_differences(model: &DiffModelFacts) -> bool { + !(model.added_columns.is_empty() + && model.removed_columns.is_empty() + && model.type_changes.is_empty()) +} + +/// Whether any model has schema or row differences. +pub(crate) fn has_findings(models: &[DiffModelFacts]) -> bool { + models.iter().any(|model| { + has_schema_differences(model) + || model.rows.as_ref().is_some_and(|rows| { + rows.unequal_count != 0 || rows.left_only_count != 0 || rows.right_only_count != 0 + }) + }) +} + +/// The explicit outcome, or `findings`/`pass` from the results. +pub(crate) fn resolved_outcome(models: &[DiffModelFacts], outcome: Option<&str>) -> String { + outcome + .filter(|outcome| !outcome.is_empty()) + .map(str::to_owned) + .unwrap_or_else(|| { + if has_findings(models) { + FINDINGS_OUTCOME + } else { + PASS_OUTCOME + } + .to_owned() + }) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs new file mode 100644 index 0000000000..c7e36b2e4f --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs @@ -0,0 +1,511 @@ +//! The subset of Rich's console layout the diff summary uses: text, groups, tables and columns. + +use crate::diff_report::_helpers::text_cells::{cell_len, wrap_line}; + +/// Text attributes and one standard foreground color, as Rich renders them. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct Style { + pub(crate) bold: bool, + pub(crate) dim: bool, + pub(crate) italic: bool, + /// The SGR foreground code, such as 36 for cyan; zero for none. + pub(crate) color: u8, +} + +impl Style { + pub(crate) const PLAIN: Self = Self { + bold: false, + dim: false, + italic: false, + color: 0, + }; + + /// This style with `other`'s attributes and color layered on top. + pub(crate) fn with(self, other: Self) -> Self { + Self { + bold: self.bold || other.bold, + dim: self.dim || other.dim, + italic: self.italic || other.italic, + color: if other.color == 0 { + self.color + } else { + other.color + }, + } + } + + fn sgr(self) -> String { + let mut codes: Vec = Vec::new(); + for (enabled, code) in [(self.bold, "1"), (self.dim, "2"), (self.italic, "3")] { + if enabled { + codes.push(code.to_owned()); + } + } + if self.color != 0 { + codes.push(self.color.to_string()); + } + codes.join(";") + } +} + +#[derive(Clone, Debug)] +pub(crate) struct Segment { + text: String, + style: Style, +} + +pub(crate) type Line = Vec; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Justify { + Left, + Center, + Right, +} + +/// Box-drawing characters per line kind, each as left, horizontal, divider and right. +#[derive(Clone, Copy, Debug)] +pub(crate) struct BoxChars { + pub(crate) top: [char; 4], + pub(crate) row: [char; 4], + pub(crate) separator: [char; 4], + pub(crate) bottom: [char; 4], +} + +#[derive(Clone, Debug)] +pub(crate) struct Column { + pub(crate) justify: Justify, + pub(crate) style: Style, + /// Maximum content cells; longer content folds. + pub(crate) max_width: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct Table { + pub(crate) columns: Vec, + pub(crate) rows: Vec>, + /// Whether a section separator follows each row. + pub(crate) section_after: Vec, + pub(crate) borders: Option, + /// Left and right cell padding. + pub(crate) padding: (usize, usize), + pub(crate) collapse_padding: bool, + pub(crate) pad_edge: bool, +} + +#[derive(Clone, Debug)] +pub(crate) enum Node { + Text(String, Style), + Group(Vec), + Table(Table), + Columns(Vec), + /// A heavy-bordered panel spanning the width with one centered title line. + Panel(String, Style), +} + +/// How a table cell justifies and overflows its text. +#[derive(Clone, Copy, Debug)] +struct CellLayout { + justify: Justify, + fold: bool, +} + +impl Node { + pub(crate) fn plain(text: impl Into) -> Self { + Self::Text(text.into(), Style::PLAIN) + } + + pub(crate) fn styled(text: impl Into, style: Style) -> Self { + Self::Text(text.into(), style) + } +} + +impl Table { + /// `Table.grid`: no borders, collapsed padding and no edge padding. + pub(crate) fn grid(columns: Vec, right_padding: usize) -> Self { + Self { + columns, + rows: Vec::new(), + section_after: Vec::new(), + borders: None, + padding: (right_padding, right_padding), + collapse_padding: true, + pad_edge: false, + } + } + + pub(crate) fn add_row(&mut self, cells: Vec) { + self.rows.push(cells); + self.section_after.push(false); + } + + pub(crate) fn add_section(&mut self) { + if let Some(last) = self.section_after.last_mut() { + *last = true; + } + } +} + +pub(crate) fn column(justify: Justify, style: Style) -> Column { + Column { + justify, + style, + max_width: None, + } +} + +fn text_lines(text: &str) -> Vec<&str> { + text.split('\n').collect() +} + +fn segment(text: String, style: Style) -> Segment { + Segment { text, style } +} + +fn line_cells(line: &Line) -> usize { + line.iter().map(|part| cell_len(&part.text)).sum() +} + +/// `Measurement.get(...).maximum` of one node within `max_width` cells. +fn measure(node: &Node, max_width: usize) -> usize { + let natural: usize = match node { + Node::Text(text, _) => text_lines(text) + .into_iter() + .map(cell_len) + .max() + .unwrap_or(0), + Node::Group(children) => children + .iter() + .map(|child| measure(child, max_width)) + .max() + .unwrap_or(0), + Node::Table(table) => { + let extra: usize = table_extra_width(table); + column_widths(table, max_width.saturating_sub(extra)) + .iter() + .sum::() + + extra + + if table.rows.is_empty() { max_width } else { 0 } + } + Node::Columns(_) | Node::Panel(..) => max_width, + }; + natural.min(max_width) +} + +fn table_extra_width(table: &Table) -> usize { + table.borders.map_or(0, |_| table.columns.len() + 1) +} + +/// The left and right padding of one cell column. +fn cell_padding(table: &Table, index: usize) -> (usize, usize) { + let (mut left, mut right) = table.padding; + if table.collapse_padding && index > 0 { + left = left.saturating_sub(right); + } + if !table.pad_edge { + if index == 0 { + left = 0; + } + if index + 1 == table.columns.len() { + right = 0; + } + } + (left, right) +} + +fn column_maximum(table: &Table, index: usize, max_width: usize) -> usize { + let (left, right) = cell_padding(table, index); + let padding: usize = left + right; + let mut widest: Option = None; + for row in &table.rows { + if let Some(cell) = row.get(index) { + let width: usize = measure(cell, max_width.saturating_sub(padding)) + padding; + widest = Some(widest.map_or(width, |current| current.max(width))); + } + } + let mut maximum: usize = widest.unwrap_or(max_width).min(max_width); + if let Some(limit) = table.columns[index].max_width { + let (configured_left, configured_right) = table.padding; + let configured: usize = if table.collapse_padding && index > 0 { + configured_left.saturating_sub(configured_right) + configured_right + } else { + configured_left + configured_right + }; + maximum = maximum.min(limit + configured); + } + maximum +} + +fn ratio_reduce( + total: usize, + ratios: &[usize], + maximums: &[usize], + values: &[usize], +) -> Vec { + let ratios: Vec = ratios + .iter() + .zip(maximums) + .map(|(ratio, maximum)| if *maximum > 0 { *ratio } else { 0 }) + .collect(); + let mut total_ratio: usize = ratios.iter().sum(); + if total_ratio == 0 { + return values.to_vec(); + } + let mut remaining: usize = total; + let mut result: Vec = Vec::with_capacity(values.len()); + for ((ratio, maximum), value) in ratios.iter().zip(maximums).zip(values) { + if *ratio > 0 && total_ratio > 0 { + let share: f64 = (*ratio as f64) * (remaining as f64) / (total_ratio as f64); + let distributed: usize = (*maximum).min(share.round_ties_even() as usize); + result.push(value.saturating_sub(distributed)); + remaining = remaining.saturating_sub(distributed); + total_ratio -= ratio; + } else { + result.push(*value); + } + } + result +} + +fn collapse_widths(mut widths: Vec, max_width: usize) -> Vec { + let mut total: usize = widths.iter().sum(); + while total > 0 && total > max_width { + let excess: usize = total - max_width; + let widest: usize = widths.iter().copied().max().unwrap_or(0); + let second: usize = widths + .iter() + .copied() + .filter(|width| *width != widest) + .max() + .unwrap_or(0); + let difference: usize = widest - second; + if difference == 0 { + break; + } + let ratios: Vec = widths + .iter() + .map(|width| usize::from(*width == widest)) + .collect(); + let maximums: Vec = vec![excess.min(difference); widths.len()]; + widths = ratio_reduce(excess, &ratios, &maximums, &widths); + total = widths.iter().sum(); + } + widths +} + +/// `Table._calculate_column_widths` for a table that does not expand. +fn column_widths(table: &Table, max_width: usize) -> Vec { + let mut widths: Vec = (0..table.columns.len()) + .map(|index| column_maximum(table, index, max_width).max(1)) + .collect(); + if widths.iter().sum::() > max_width { + widths = collapse_widths(widths, max_width); + let total: usize = widths.iter().sum(); + if total > max_width { + let ones: Vec = vec![1; widths.len()]; + widths = ratio_reduce(total - max_width, &ones, &widths.clone(), &widths); + } + widths = widths + .iter() + .enumerate() + .map(|(index, width)| column_maximum(table, index, *width)) + .collect(); + } + widths +} + +fn justified(text: String, width: usize, justify: Justify) -> String { + let content: String = match justify { + Justify::Left => text, + Justify::Center | Justify::Right => text.trim_end().to_owned(), + }; + let used: usize = cell_len(&content); + let free: usize = width.saturating_sub(used); + match justify { + Justify::Left => content + &" ".repeat(free), + Justify::Center => " ".repeat(free / 2) + &content + &" ".repeat(free - free / 2), + Justify::Right => " ".repeat(free) + &content, + } +} + +/// Render a node into lines at most `width` cells wide. +fn render(node: &Node, width: usize, layout: Option, base: Style) -> Vec { + match node { + Node::Text(text, style) => { + let combined: Style = base.with(*style); + let fold: bool = layout.is_none_or(|cell| cell.fold); + let mut lines: Vec = Vec::new(); + for source in text_lines(text) { + for piece in wrap_line(source, width, fold) { + let piece: String = match layout { + Some(cell) => justified(piece, width, cell.justify), + None => piece, + }; + lines.push(vec![segment(piece, combined)]); + } + } + lines + } + Node::Group(children) => children + .iter() + .flat_map(|child| render(child, width, layout, base)) + .collect(), + Node::Table(table) => render_table(table, width), + Node::Panel(title, style) => { + let inner: usize = width.saturating_sub(4); + vec![ + border_line(['┏', '━', '━', '┓'], &[width.saturating_sub(2)]), + vec![ + segment("┃ ".to_owned(), Style::PLAIN), + segment(justified(title.clone(), inner, Justify::Center), *style), + segment(" ┃".to_owned(), Style::PLAIN), + ], + border_line(['┗', '━', '━', '┛'], &[width.saturating_sub(2)]), + ] + } + Node::Columns(items) => render_table(&columns_grid(items, width), width), + } +} + +/// The grid `Columns` lays its items out in. +fn columns_grid(items: &[Node], max_width: usize) -> Table { + let widths: Vec = items.iter().map(|item| measure(item, max_width)).collect(); + let mut count: usize = items.len().max(1); + while count > 1 { + let mut column_widths: Vec = Vec::new(); + let mut overflow: bool = false; + for (position, width) in widths.iter().enumerate() { + let column_index: usize = position % count; + if column_widths.len() <= column_index { + column_widths.push(0); + } + column_widths[column_index] = column_widths[column_index].max(*width); + if column_widths.iter().sum::() > max_width { + count = column_widths.len() - 1; + overflow = true; + break; + } + } + if !overflow { + break; + } + } + let mut grid: Table = Table::grid(vec![column(Justify::Left, Style::PLAIN); count], 0); + for chunk in items.chunks(count) { + let mut cells: Vec = chunk.to_vec(); + cells.resize(count, Node::plain("")); + grid.add_row(cells); + } + grid +} + +fn border_line(chars: [char; 4], widths: &[usize]) -> Line { + let [left, horizontal, divider, right] = chars; + let mut text: String = left.to_string(); + for (index, width) in widths.iter().enumerate() { + if index > 0 { + text.push(divider); + } + text.push_str(&horizontal.to_string().repeat(*width)); + } + text.push(right); + vec![segment(text, Style::PLAIN)] +} + +fn render_cell(table: &Table, index: usize, cell: &Node, width: usize) -> Vec { + let spec: &Column = &table.columns[index]; + let (left, right) = cell_padding(table, index); + let content_width: usize = width.saturating_sub(left + right); + let mut lines: Vec = Vec::new(); + let layout: CellLayout = CellLayout { + justify: spec.justify, + fold: spec.max_width.is_some(), + }; + for mut line in render(cell, content_width, Some(layout), spec.style) { + let used: usize = line_cells(&line); + let mut padded: Line = Vec::new(); + if left > 0 { + padded.push(segment(" ".repeat(left), spec.style)); + } + padded.append(&mut line); + if used < content_width { + padded.push(segment(" ".repeat(content_width - used), spec.style)); + } + if right > 0 { + padded.push(segment(" ".repeat(right), spec.style)); + } + lines.push(padded); + } + lines +} + +fn render_table(table: &Table, max_width: usize) -> Vec { + let extra: usize = table_extra_width(table); + let widths: Vec = column_widths(table, max_width.saturating_sub(extra)); + let mut lines: Vec = Vec::new(); + if let Some(chars) = table.borders { + lines.push(border_line(chars.top, &widths)); + } + for (row_index, row) in table.rows.iter().enumerate() { + let empty: Node = Node::plain(""); + let cells: Vec> = widths + .iter() + .enumerate() + .map(|(index, width)| { + render_cell(table, index, row.get(index).unwrap_or(&empty), *width) + }) + .collect(); + let height: usize = cells.iter().map(Vec::len).max().unwrap_or(0); + for line_index in 0..height { + let mut line: Line = Vec::new(); + if let Some(chars) = table.borders { + line.push(segment(chars.row[0].to_string(), Style::PLAIN)); + } + for (index, cell) in cells.iter().enumerate() { + if let (true, Some(chars)) = (index > 0, table.borders) { + line.push(segment(chars.row[2].to_string(), Style::PLAIN)); + } + match cell.get(line_index) { + Some(content) => line.extend(content.iter().cloned()), + None => line.push(segment( + " ".repeat(widths[index]), + table.columns[index].style, + )), + } + } + if let Some(chars) = table.borders { + line.push(segment(chars.row[3].to_string(), Style::PLAIN)); + } + lines.push(line); + } + let separated: bool = table.section_after[row_index] && row_index + 1 < table.rows.len(); + if let (true, Some(chars)) = (separated, table.borders) { + lines.push(border_line(chars.separator, &widths)); + } + } + if let Some(chars) = table.borders { + lines.push(border_line(chars.bottom, &widths)); + } + lines +} + +/// Render top-level nodes as `Console.print` calls do, one after another. +pub(crate) fn print_nodes(nodes: &[Node], width: usize, use_color: bool) -> String { + let mut output: String = String::new(); + for node in nodes { + for line in render(node, width, None, Style::PLAIN) { + for part in line { + if part.text.is_empty() { + continue; + } + if use_color && part.style != Style::PLAIN { + output.push_str(&format!("\x1b[{}m{}\x1b[0m", part.style.sgr(), part.text)); + } else { + output.push_str(&part.text); + } + } + output.push('\n'); + } + } + output.trim_end().to_owned() +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs new file mode 100644 index 0000000000..7a2971b107 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod evidence; +pub(crate) mod findings; +pub(crate) mod layout; +pub(crate) mod numbers; +pub(crate) mod payload; +pub(crate) mod request_rules; +pub(crate) mod sections; +pub(crate) mod shell_words; +pub(crate) mod size_guard; +pub(crate) mod text_cells; diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/numbers.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/numbers.rs new file mode 100644 index 0000000000..249e7a7864 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/numbers.rs @@ -0,0 +1,31 @@ +//! Python's `{:,}` and `{:.2%}` number formats. + +/// An integer with comma thousands separators. +pub(crate) fn grouped(value: i64) -> String { + let digits: String = value.unsigned_abs().to_string(); + let mut result: String = String::new(); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + result.push(','); + } + result.push(digit); + } + if value < 0 { + format!("-{result}") + } else { + result + } +} + +/// A ratio as a percentage with two decimals. +pub(crate) fn percent(ratio: f64) -> String { + format!("{:.2}%", ratio * 100.0) +} + +/// `numerator / denominator` as a percentage, or `0.00%` without a denominator. +pub(crate) fn percentage(numerator: i64, denominator: i64) -> String { + if denominator == 0 { + return "0.00%".to_owned(); + } + percent(numerator as f64 / denominator as f64) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/payload.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/payload.rs new file mode 100644 index 0000000000..c46ae03d65 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/payload.rs @@ -0,0 +1,346 @@ +//! The structured JSON diff document, before serialization. + +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::diff_report::_helpers::evidence::render_example_pair; +use crate::diff_report::_helpers::findings::resolved_outcome; +use crate::diff_report::constants::{FINDINGS_OUTCOME, PASS_OUTCOME, QUERY_INPUT_KIND}; +use crate::diff_report::models::{ + DiffCoverageFacts, DiffDocumentOptions, DiffExampleOptions, DiffModelFacts, DiffValue, + RenderedValue, +}; +use crate::diff_report::types::DiffNamedValue; + +pub(crate) fn integer(value: i64) -> JsonValue { + JsonInteger::parse(&value.to_string()).map_or(JsonValue::Null, JsonValue::Integer) +} + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +pub(crate) fn object(entries: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + entries + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + ) +} + +fn strings(values: &[String]) -> JsonValue { + JsonValue::Array(values.iter().map(|value| text(value)).collect()) +} + +fn optional(value: Option<&DiffValue>) -> JsonValue { + value.map_or(JsonValue::Null, |value| value.json.clone()) +} + +/// `{name: value}` where a repeated name keeps its first position and last value. +fn named_values(values: &[DiffNamedValue]) -> JsonValue { + let mut entries: Vec<(String, JsonValue)> = Vec::new(); + for (name, value) in values { + match entries.iter_mut().find(|(existing, _)| existing == name) { + Some(entry) => entry.1 = value.json.clone(), + None => entries.push((name.clone(), value.json.clone())), + } + } + JsonValue::Object(entries) +} + +fn rendered(value: &RenderedValue) -> JsonValue { + object(vec![ + ("value", value.text.as_deref().map_or(JsonValue::Null, text)), + ("original_length", integer(value.original_length as i64)), + ("truncated", JsonValue::Bool(value.truncated)), + ("suppressed", JsonValue::Bool(value.suppressed)), + ( + "first_difference", + value + .first_difference + .map_or(JsonValue::Null, |index| integer(index as i64)), + ), + ]) +} + +fn coverage(coverage: Option<&DiffCoverageFacts>) -> JsonValue { + coverage.map_or(JsonValue::Null, |coverage| { + object(vec![ + ("row_count", integer(coverage.row_count)), + ("minimum_cursor", optional(coverage.minimum_cursor.as_ref())), + ("maximum_cursor", optional(coverage.maximum_cursor.as_ref())), + ]) + }) +} + +fn examples(model: &DiffModelFacts, options: DiffExampleOptions) -> JsonValue { + let mut values_truncated: bool = false; + let mut unequal_rows: Vec = Vec::new(); + for sample in &model.unequal_samples { + let mut cells: Vec = Vec::new(); + for cell in &sample.changed_cells { + let (left, right) = render_example_pair(&cell.left.text, &cell.right.text, options); + values_truncated = values_truncated || left.truncated || right.truncated; + cells.push(object(vec![ + ("column", text(&cell.name)), + ("from", rendered(&left)), + ("to", rendered(&right)), + ])); + } + unequal_rows.push(object(vec![ + ("key", named_values(&sample.key_values)), + ("changed_cells", JsonValue::Array(cells)), + ])); + } + let truncated_by_count: bool = model.rows.as_ref().is_some_and(|rows| { + rows.unequal_count > model.unequal_samples.len() as i64 + || rows.left_only_count > model.left_only_keys.len() as i64 + || rows.right_only_count > model.right_only_keys.len() as i64 + }); + let keys = |samples: &[Vec]| { + JsonValue::Array(samples.iter().map(|sample| named_values(sample)).collect()) + }; + object(vec![ + ("unequal_rows", JsonValue::Array(unequal_rows)), + ("from_only_keys", keys(&model.left_only_keys)), + ("to_only_keys", keys(&model.right_only_keys)), + ("truncated_by_count", JsonValue::Bool(truncated_by_count)), + ("values_truncated", JsonValue::Bool(values_truncated)), + ( + "values_suppressed", + JsonValue::Bool(options.suppress_values), + ), + ( + "max_value_length", + options + .max_value_length + .map_or(JsonValue::Null, |limit| integer(limit as i64)), + ), + ]) +} + +fn model_payload(model: &DiffModelFacts, options: &DiffDocumentOptions) -> JsonValue { + let type_changes: Vec = model + .type_changes + .iter() + .map(|(name, from_type, to_type)| { + object(vec![ + ("name", text(name)), + ("from_type", text(from_type)), + ("to_type", text(to_type)), + ]) + }) + .collect(); + let mut entries: Vec<(&str, JsonValue)> = vec![ + ("input_kind", text(&model.input_kind)), + ("name", text(&model.name)), + ("from_label", text(&options.from_label)), + ("to_label", text(&options.to_label)), + ("from_relation", text(&model.left_relation)), + ("to_relation", text(&model.right_relation)), + ("unique_key", strings(&model.unique_key)), + ("unkeyed", JsonValue::Bool(model.unkeyed)), + ("excluded_columns", strings(&model.excluded_columns)), + ( + "schema", + object(vec![ + ("added_columns", strings(&model.added_columns)), + ("removed_columns", strings(&model.removed_columns)), + ("type_changed_columns", JsonValue::Array(type_changes)), + ]), + ), + ]; + let Some(rows) = &model.rows else { + entries.push(("comparison_scope", text("schema_only"))); + return object(entries); + }; + let keyed = |value: i64| { + if model.unkeyed { + JsonValue::Null + } else { + integer(value) + } + }; + let changed: Vec<(String, JsonValue)> = rows + .columns + .iter() + .map(|column| (column.name.clone(), integer(column.mismatched_count))) + .collect(); + let mismatches: Vec = rows + .columns + .iter() + .filter(|column| column.mismatched_count > 0) + .map(|column| { + object(vec![ + ("column", text(&column.name)), + ("rows", integer(column.mismatched_count)), + ]) + }) + .collect(); + entries.extend([ + ( + "comparison_scope", + text(if rows.exhaustive { + "exhaustive" + } else { + "sampled" + }), + ), + ( + "coverage", + object(vec![ + ( + "requested", + object(vec![ + ( + "cursor_column", + rows.cursor_column.as_deref().map_or(JsonValue::Null, text), + ), + ("start", optional(rows.cursor_start.as_ref())), + ("end", optional(rows.cursor_end.as_ref())), + ]), + ), + ("from", coverage(rows.left_coverage.as_ref())), + ("to", coverage(rows.right_coverage.as_ref())), + ( + "differs", + JsonValue::Bool(rows.cursor_column.is_some() && rows.coverage_differs), + ), + ]), + ), + ( + "sampling", + object(vec![ + ( + "configured_row_limit", + rows.sampling + .map_or(JsonValue::Null, |(limit, _)| integer(limit)), + ), + ( + "seed", + rows.sampling + .map_or(JsonValue::Null, |(_, seed)| integer(seed)), + ), + ("population_keys", keyed(rows.population_count)), + ("compared_keys", keyed(rows.compared_count)), + ( + "coverage_ratio", + JsonValue::Float(if rows.population_count == 0 { + 1.0 + } else { + rows.compared_count as f64 / rows.population_count as f64 + }), + ), + ("exhaustive", JsonValue::Bool(rows.exhaustive)), + ]), + ), + ( + "rows", + object(vec![ + ("from_count", integer(rows.left_count)), + ("to_count", integer(rows.right_count)), + ( + "joined_count", + integer(rows.equal_count + rows.unequal_count), + ), + ("equal_count", integer(rows.equal_count)), + ("unequal_count", integer(rows.unequal_count)), + ("from_only_count", integer(rows.left_only_count)), + ("to_only_count", integer(rows.right_only_count)), + ("changed_columns", JsonValue::Object(unique(changed))), + ("column_mismatches", JsonValue::Array(mismatches)), + ]), + ), + ("examples", examples(model, options.examples)), + ]); + object(entries) +} + +fn unique(entries: Vec<(String, JsonValue)>) -> Vec<(String, JsonValue)> { + let mut result: Vec<(String, JsonValue)> = Vec::new(); + for (key, value) in entries { + match result.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1 = value, + None => result.push((key, value)), + } + } + result +} + +/// Python's `round(value, 6)`. +fn round_six(value: f64) -> f64 { + format!("{value:.6}").parse::().unwrap_or(value) +} + +/// The complete diff document. +pub(crate) fn document(models: &[DiffModelFacts], options: &DiffDocumentOptions) -> JsonValue { + let payloads: Vec = models + .iter() + .map(|model| model_payload(model, options)) + .collect(); + let outcome: String = resolved_outcome(models, options.outcome.as_deref()); + let status: &str = match outcome.as_str() { + PASS_OUTCOME => "no_differences_found", + FINDINGS_OUTCOME => "differences_found", + other => other, + }; + let query_payloads: Vec = models + .iter() + .zip(&payloads) + .filter(|(model, _)| model.input_kind == QUERY_INPUT_KIND) + .map(|(_, payload)| payload.clone()) + .collect(); + let mut entries: Vec<(&str, JsonValue)> = vec![ + ("schema_version", integer(1)), + ("status", text(status)), + ("outcome", text(&outcome)), + ("from", text(&options.from_label)), + ("to", text(&options.to_label)), + ("models", JsonValue::Array(payloads)), + ]; + if !query_payloads.is_empty() { + entries.push(("input_kind", text(QUERY_INPUT_KIND))); + entries.push(("query_comparisons", JsonValue::Array(query_payloads))); + } + if let Some(phases) = &options.phase_seconds { + let mut sorted: Vec<(String, f64)> = phases.clone(); + sorted.sort_by(|a, b| a.0.cmp(&b.0)); + entries.push(( + "phase_seconds", + JsonValue::Object( + sorted + .into_iter() + .map(|(phase, seconds)| (phase, JsonValue::Float(round_six(seconds)))) + .collect(), + ), + )); + } + json_safe(object(entries)) +} + +/// Non-finite floats as `{"kind": "non_finite_number", "value": ...}` objects. +fn json_safe(value: JsonValue) -> JsonValue { + match value { + JsonValue::Float(number) if !number.is_finite() => { + let label: &str = if number.is_nan() { + "NaN" + } else if number.is_sign_positive() { + "Infinity" + } else { + "-Infinity" + }; + object(vec![ + ("kind", text("non_finite_number")), + ("value", text(label)), + ]) + } + JsonValue::Array(items) => JsonValue::Array(items.into_iter().map(json_safe).collect()), + JsonValue::Object(entries) => JsonValue::Object( + entries + .into_iter() + .map(|(key, item)| (key, json_safe(item))) + .collect(), + ), + other => other, + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/request_rules.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/request_rules.rs new file mode 100644 index 0000000000..489ddadb55 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/request_rules.rs @@ -0,0 +1,209 @@ +//! The individual `sqb diff` flag rules. + +use crate::diff_report::errors::DiffUserError; +use crate::diff_report::models::DiffRequestFacts; + +pub(crate) fn user_error(message: &str, code: &str) -> DiffUserError { + DiffUserError { + message: message.to_owned(), + code: code.to_owned(), + help: None, + } +} + +fn not_positive(value: Option) -> bool { + value.is_some_and(|value| value <= 0) +} + +fn blank(value: Option<&String>) -> bool { + value.is_some_and(|value| value.trim().is_empty()) +} + +/// The first violated rule that applies to model and raw-query diffs alike. +pub(crate) fn common_violation(request: &DiffRequestFacts) -> Option<(&'static str, &'static str)> { + let modes: usize = usize::from(request.full) + + usize::from(request.schema_only) + + usize::from(request.bounded.is_some()); + let mut lowered: Vec = request + .unique_key_override + .iter() + .map(|key| key.to_lowercase()) + .collect(); + lowered.sort(); + let total: usize = lowered.len(); + lowered.dedup(); + let rules: [(bool, &str, &str); 15] = [ + ( + modes > 1, + "diff accepts at most one of --full, --schema-only, or --bounded", + "C201", + ), + ( + not_positive(request.max_column_examples), + "diff --max-column-examples must be positive", + "C202", + ), + ( + not_positive(request.max_row_only_examples), + "diff --max-row-only-examples must be positive", + "C203", + ), + ( + not_positive(request.sample_rows), + "diff --sample-rows must be positive", + "C208", + ), + ( + request.exhaustive && request.sample_rows.is_some(), + "diff --exhaustive cannot be combined with --sample-rows", + "C209", + ), + ( + not_positive(request.max_models), + "diff --max-models must be positive", + "C210", + ), + ( + not_positive(request.max_columns), + "diff --max-columns must be positive", + "C211", + ), + ( + not_positive(request.max_value_length), + "diff --max-value-length must be positive", + "C246", + ), + ( + request.full_example_values && request.max_value_length.is_some(), + "diff --full-example-values cannot be combined with --max-value-length", + "C247", + ), + ( + request.full_example_values && request.suppress_example_values, + "diff --full-example-values cannot be combined with --no-example-values", + "C248", + ), + ( + request.unkeyed && !request.unique_key_override.is_empty(), + "diff --unkeyed cannot be combined with --key", + "C226", + ), + ( + request.unkeyed && (request.sample_rows.is_some() || request.sample_seed.is_some()), + "unkeyed diff does not support sampling", + "C222", + ), + ( + request.unkeyed && !request.tolerance_overrides.is_empty(), + "unkeyed diff does not support tolerances", + "C223", + ), + ( + request + .unique_key_override + .iter() + .any(|key| key.trim().is_empty()), + "diff --key values must not be empty", + "C241", + ), + ( + lowered.len() != total, + "diff --key values must be unique", + "C242", + ), + ]; + rules + .into_iter() + .find(|(violated, _, _)| *violated) + .map(|(_, message, code)| (message, code)) +} + +/// The first violated rule specific to raw-query diffs. +pub(crate) fn query_violation(request: &DiffRequestFacts) -> Option<(&'static str, &'static str)> { + let left: bool = request.left_query || request.left_query_file; + let right: bool = request.right_query || request.right_query_file; + let keyed: bool = !request.unique_key_override.is_empty(); + let rules: [(bool, &str, &str); 12] = [ + ( + request.from_name.is_some() || request.to_name.is_some(), + "raw-query diff does not accept FROM:TO", + "C212", + ), + ( + !request.select.is_empty() || !request.exclude.is_empty(), + "raw-query diff does not accept model selectors", + "C213", + ), + ( + request.bounded.is_some(), + "raw-query diff does not support --bounded; filter both raw queries explicitly", + "C214", + ), + ( + request.max_models.is_some(), + "raw-query diff does not accept --max-models", + "C243", + ), + ( + blank(request.left_label.as_ref()), + "raw-query diff --left-label must not be empty", + "C249", + ), + ( + blank(request.right_label.as_ref()), + "raw-query diff --right-label must not be empty", + "C250", + ), + (!left, "raw-query diff requires a left query", "C216"), + ( + request.left_query && request.left_query_file, + "raw-query diff accepts either --left-query or --left-query-file, not both", + "C217", + ), + (!right, "raw-query diff requires a right query", "C218"), + ( + request.right_query && request.right_query_file, + "raw-query diff accepts either --right-query or --right-query-file, not both", + "C219", + ), + ( + request.schema_only && (request.unkeyed || keyed), + "schema-only raw-query diff does not accept --key or --unkeyed", + "C220", + ), + ( + !request.schema_only && keyed == request.unkeyed, + "raw-query diff requires one or more --key values or explicit --unkeyed", + "C221", + ), + ]; + rules + .into_iter() + .find(|(violated, _, _)| *violated) + .map(|(_, message, code)| (message, code)) +} + +/// The first violated rule specific to model diffs. +pub(crate) fn model_violation(request: &DiffRequestFacts) -> Option<(&'static str, &'static str)> { + let rules: [(bool, &str, &str); 3] = [ + ( + request.from_name.is_none(), + "model diff requires FROM or FROM:TO", + "C224", + ), + ( + request.selected_target.is_some(), + "model diff does not accept --target", + "C225", + ), + ( + request.left_label.is_some() || request.right_label.is_some(), + "model diff does not accept query labels", + "C251", + ), + ]; + rules + .into_iter() + .find(|(violated, _, _)| *violated) + .map(|(_, message, code)| (message, code)) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/sections.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/sections.rs new file mode 100644 index 0000000000..ba51c25674 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/sections.rs @@ -0,0 +1,633 @@ +//! The sections of the terminal diff summary, as layout nodes. + +use std::collections::BTreeMap; + +use crate::diff_report::_helpers::evidence::{ + display_example_value, render_example_pair, render_key_value, +}; +use crate::diff_report::_helpers::findings::has_schema_differences; +use crate::diff_report::_helpers::layout::{BoxChars, Column, Justify, Node, Style, Table, column}; +use crate::diff_report::_helpers::numbers::{grouped, percent, percentage}; +use crate::diff_report::constants::{ + CHANGED_COLUMN_NAME_WIDTH, QUERY_INPUT_KIND, TOP_CHANGED_COLUMNS, +}; +use crate::diff_report::models::{ + DiffColumnFacts, DiffCoverageFacts, DiffExampleOptions, DiffModelFacts, DiffRowFacts, + DiffTextOptions, +}; +use crate::diff_report::types::DiffNamedValue; + +pub(crate) const BOLD: Style = Style { + bold: true, + ..Style::PLAIN +}; +pub(crate) const OBJECT: Style = Style { + bold: true, + color: 36, + ..Style::PLAIN +}; +pub(crate) const MUTED: Style = Style { + dim: true, + ..Style::PLAIN +}; +const ITALIC: Style = Style { + italic: true, + ..Style::PLAIN +}; +const ADDED: Style = Style { + color: 32, + ..Style::PLAIN +}; +const REMOVED: Style = Style { + color: 31, + ..Style::PLAIN +}; +const WARNING: Style = Style { + color: 33, + ..Style::PLAIN +}; + +const HEAVY_EDGE: BoxChars = BoxChars { + top: ['┏', '━', '┯', '┓'], + row: ['┃', ' ', '│', '┃'], + separator: ['┠', '─', '┼', '┨'], + bottom: ['┗', '━', '┷', '┛'], +}; +const ROUNDED: BoxChars = BoxChars { + top: ['╭', '─', '┬', '╮'], + row: ['│', ' ', '│', '│'], + separator: ['├', '─', '┼', '┤'], + bottom: ['╰', '─', '┴', '╯'], +}; +const SQUARE: BoxChars = BoxChars { + top: ['┌', '─', '┬', '┐'], + row: ['│', ' ', '│', '│'], + separator: ['├', '─', '┼', '┤'], + bottom: ['└', '─', '┴', '┘'], +}; + +/// Everything one model section needs besides the model itself. +pub(crate) struct SectionContext<'a> { + pub(crate) from_label: &'a str, + pub(crate) to_label: &'a str, + pub(crate) text: &'a DiffTextOptions, + pub(crate) examples: DiffExampleOptions, +} + +fn boxed(columns: Vec, borders: BoxChars, padding: (usize, usize)) -> Table { + Table { + columns, + rows: Vec::new(), + section_after: Vec::new(), + borders: Some(borders), + padding, + collapse_padding: false, + pad_edge: true, + } +} + +fn texts(values: &[String]) -> Vec { + values + .iter() + .map(|value| Node::plain(value.clone())) + .collect() +} + +/// The printed nodes of one model: overview and every section. +pub(crate) fn model_nodes(model: &DiffModelFacts, context: &SectionContext<'_>) -> Vec { + let mut nodes: Vec = vec![overview(model, &context.text.mode_label), Node::plain("")]; + nodes.extend(section("Schemas", schema_summary(model))); + let Some(rows) = &model.rows else { + return nodes; + }; + nodes.extend(section("Coverage", coverage(rows, context))); + nodes.extend(section( + if rows.exhaustive { + "Rows" + } else { + "Sampled Rows" + }, + row_counts(rows, context), + )); + nodes.extend(section( + "Changed Columns", + changed_columns(model, rows, context), + )); + let verbose_examples: Option = + if context.text.verbose && !model.unequal_samples.is_empty() { + examples(model, context, None) + } else { + None + }; + if let Some(content) = verbose_examples { + nodes.extend(section("Examples", content)); + } + for (label, samples) in [ + (context.from_label, &model.left_only_keys), + (context.to_label, &model.right_only_keys), + ] { + if !samples.is_empty() { + nodes.extend(section( + &format!("{label} only"), + side_only(label, samples, context.text.max_row_only_examples), + )); + } + } + nodes +} + +fn section(title: &str, content: Node) -> [Node; 4] { + [ + Node::styled(title, BOLD), + Node::styled("▔".repeat(title.chars().count()), BOLD), + content, + Node::plain(""), + ] +} + +fn overview(model: &DiffModelFacts, mode_label: &str) -> Node { + let mut table: Table = Table::grid( + vec![ + column(Justify::Right, OBJECT), + column(Justify::Left, Style::PLAIN), + ], + 2, + ); + let mut add = |label: &str, value: String| { + table.add_row(vec![Node::plain(label), Node::plain(value)]); + }; + add( + if model.input_kind == QUERY_INPUT_KIND { + "Input" + } else { + "Model" + }, + model.name.clone(), + ); + add( + "Key", + if model.unique_key.is_empty() { + "".to_owned() + } else { + model.unique_key.join(", ") + }, + ); + add( + "Comparison", + if model.bounded_fallback { + format!("{mode_label} (fallback to full row diff)") + } else { + mode_label.to_owned() + }, + ); + if model.bounded_fallback { + add( + "Fallback", + "no cursor configured; used full row diff".to_owned(), + ); + } + if !model.excluded_columns.is_empty() { + add("Excluded", model.excluded_columns.join(", ")); + } + if let Some(rows) = &model.rows { + add("Scope", row_scope(rows, model.unkeyed)); + if let Some(label) = tolerance_label(&rows.columns) { + add("Tolerances", label); + } + } + Node::Table(table) +} + +fn tolerance_label(columns: &[DiffColumnFacts]) -> Option { + let labels: Vec = columns + .iter() + .filter_map(|column| { + let (absolute, relative) = column.tolerance.as_ref()?; + let mut parts: Vec = Vec::new(); + if let Some(value) = absolute { + parts.push(format!("absolute={value}")); + } + if let Some(value) = relative { + parts.push(format!("relative={value}")); + } + Some(format!("{} {}", column.name, parts.join(" "))) + }) + .collect(); + (!labels.is_empty()).then(|| labels.join(", ")) +} + +fn row_scope(rows: &DiffRowFacts, unkeyed: bool) -> String { + if unkeyed { + return format!( + "exhaustive ({} multiset rows)", + grouped(rows.population_count) + ); + } + let Some((row_limit, seed)) = rows.sampling else { + return format!("exhaustive ({} union keys)", grouped(rows.population_count)); + }; + if rows.exhaustive { + return format!( + "exhaustive ({} union keys; configured sample limit {})", + grouped(rows.compared_count), + grouped(row_limit) + ); + } + format!( + "sampled {} of {} union keys ({}; seed {seed})", + grouped(rows.compared_count), + grouped(rows.population_count), + percentage(rows.compared_count, rows.population_count) + ) +} + +fn cursor_extent(rows: &DiffRowFacts, coverage: &DiffCoverageFacts) -> String { + if rows.cursor_column.is_none() { + return "".to_owned(); + } + let text = |value: &Option<_>, missing: &str| -> String { + value.as_ref().map_or_else( + || missing.to_owned(), + |value: &crate::diff_report::models::DiffValue| value.text.clone(), + ) + }; + format!( + "{} to {} (requested {} to {})", + text(&coverage.minimum_cursor, ""), + text(&coverage.maximum_cursor, ""), + text(&rows.cursor_start, "unbounded"), + text(&rows.cursor_end, "unbounded") + ) +} + +fn coverage(rows: &DiffRowFacts, context: &SectionContext<'_>) -> Node { + let (Some(left), Some(right)) = (&rows.left_coverage, &rows.right_coverage) else { + return Node::styled("Coverage was not collected.", ITALIC); + }; + let mut table: Table = Table::grid( + vec![ + column(Justify::Left, OBJECT), + column(Justify::Right, Style::PLAIN), + column(Justify::Left, Style::PLAIN), + ], + 2, + ); + table.add_row(texts(&[ + "Side".to_owned(), + "Rows".to_owned(), + "Cursor extent".to_owned(), + ])); + for (label, side) in [(context.from_label, left), (context.to_label, right)] { + table.add_row(texts(&[ + label.to_owned(), + grouped(side.row_count), + cursor_extent(rows, side), + ])); + } + let mut nodes: Vec = vec![Node::Table(table)]; + if rows.cursor_column.is_some() && rows.coverage_differs { + nodes.push(Node::plain("")); + nodes.push(Node::styled( + "Warning: cursor coverage differs between sides; comparison continued without \ + narrowing the requested bounds.", + WARNING, + )); + } + Node::Group(nodes) +} + +fn schema_summary(model: &DiffModelFacts) -> Node { + if !has_schema_differences(model) { + return Node::styled("No schema differences.", ITALIC); + } + let count = |items: usize| grouped(items as i64); + let mut lines: Vec = vec![Node::plain(format!( + "schema differences: {}", + count(model.added_columns.len() + model.removed_columns.len() + model.type_changes.len()) + ))]; + for (label, items) in [ + ("added columns", model.added_columns.len()), + ("removed columns", model.removed_columns.len()), + ("type changes", model.type_changes.len()), + ] { + if items > 0 { + lines.push(Node::plain(format!("{label}: {}", count(items)))); + } + } + Node::Group(lines) +} + +fn delta_label(left: i64, right: i64) -> String { + if left == 0 { + return "(no baseline)".to_owned(); + } + if left == right { + return "(no change)".to_owned(); + } + let ratio: f64 = right as f64 / left as f64; + if right > left { + format!("(+{})", percent(ratio - 1.0)) + } else { + format!("(-{})", percent(1.0 - ratio)) + } +} + +fn venn_box(rows: &[(Node, bool)]) -> Node { + let mut table: Table = boxed( + vec![column(Justify::Left, Style::PLAIN); 5], + HEAVY_EDGE, + (0, 0), + ); + for (cell, section_after) in rows { + table.add_row(vec![cell.clone(); 5]); + if *section_after { + table.add_section(); + } + } + Node::Table(table) +} + +fn row_counts(rows: &DiffRowFacts, context: &SectionContext<'_>) -> Node { + let center: Column = column(Justify::Center, Style::PLAIN); + let mut counts: Table = Table::grid(vec![center.clone(), center.clone(), center], 2); + counts.add_row(texts(&[ + format!("{} count", context.from_label), + String::new(), + format!("{} count", context.to_label), + ])); + counts.add_row(texts(&[ + grouped(rows.left_count), + delta_label(rows.left_count, rows.right_count), + grouped(rows.right_count), + ])); + let joined: i64 = rows.equal_count + rows.unequal_count; + let blank: Node = Node::plain(" "); + let mut left_rows: Vec<(Node, bool)> = Vec::new(); + let mut right_rows: Vec<(Node, bool)> = Vec::new(); + if rows.left_only_count > 0 { + left_rows.push((Node::styled("-", REMOVED), true)); + } + if rows.equal_count > 0 { + left_rows.push((blank.clone(), true)); + right_rows.push((blank.clone(), true)); + } + if rows.unequal_count > 0 { + left_rows.push((blank.clone(), false)); + right_rows.push((blank, true)); + } + if rows.right_only_count > 0 { + right_rows.push((Node::styled("+", ADDED), false)); + } + let shifted: bool = rows.left_only_count > 0; + let right_box: Node = venn_box(&right_rows); + let diagram: Vec = vec![ + venn_box(&left_rows), + row_separator(rows), + if shifted { + Node::Group(vec![Node::plain("\n"), right_box]) + } else { + right_box + }, + row_stats(rows, joined, context), + joined_annotation(rows, joined), + ]; + Node::Group(vec![ + Node::Table(counts), + Node::plain(""), + Node::Columns(diagram), + Node::plain(""), + Node::plain(format!("joined: {}", grouped(joined))), + ]) +} + +fn row_separator(rows: &DiffRowFacts) -> Node { + let mut parts: Vec = Vec::new(); + if rows.left_only_count > 0 { + parts.push(Node::plain("\n")); + } + for (count, symbol) in [(rows.equal_count, " = "), (rows.unequal_count, " ≠ ")] { + if count > 0 { + parts.push(Node::plain("╌╌╌")); + parts.push(Node::styled(symbol, BOLD)); + } + } + parts.push(Node::plain("╌╌╌")); + Node::Group(parts) +} + +fn row_stats(rows: &DiffRowFacts, joined: i64, context: &SectionContext<'_>) -> Node { + let mut table: Table = boxed( + vec![ + column(Justify::Right, Style::PLAIN), + column(Justify::Left, Style::PLAIN), + column(Justify::Right, Style::PLAIN), + ], + ROUNDED, + (1, 0), + ); + let entries: [(i64, String, i64); 4] = [ + ( + rows.left_only_count, + format!("{} only", context.from_label), + rows.left_count, + ), + (rows.equal_count, "equal".to_owned(), joined), + (rows.unequal_count, "unequal".to_owned(), joined), + ( + rows.right_only_count, + format!("{} only", context.to_label), + rows.right_count, + ), + ]; + for (count, label, denominator) in entries { + table.add_row(texts(&[ + grouped(count), + label, + format!("({})", percentage(count, denominator)), + ])); + table.add_section(); + } + Node::Table(table) +} + +fn joined_annotation(rows: &DiffRowFacts, joined: i64) -> Node { + if joined == 0 { + return Node::plain(""); + } + let mut lines: Vec = Vec::new(); + if rows.left_only_count > 0 { + lines.push(Node::plain("\n")); + } + lines.push(Node::plain("╌╮")); + lines.push(Node::plain(" │")); + if rows.equal_count > 0 && rows.unequal_count > 0 { + lines.push(Node::plain(format!("╌├╴ {} joined", grouped(joined)))); + lines.push(Node::plain(" │")); + } + lines.push(Node::plain("╌╯")); + Node::Group(lines) +} + +fn changed_columns( + model: &DiffModelFacts, + rows: &DiffRowFacts, + context: &SectionContext<'_>, +) -> Node { + let mut mismatched: Vec<&DiffColumnFacts> = rows + .columns + .iter() + .filter(|column| column.mismatched_count > 0) + .collect(); + mismatched.sort_by(|a, b| { + b.mismatched_count + .cmp(&a.mismatched_count) + .then_with(|| a.name.cmp(&b.name)) + }); + if mismatched.is_empty() { + return Node::styled( + if rows.exhaustive { + "No changed columns." + } else { + "No changed columns in sampled keys." + }, + ITALIC, + ); + } + let mut name_column: Column = column(Justify::Left, OBJECT); + name_column.max_width = Some(CHANGED_COLUMN_NAME_WIDTH); + let mut table: Table = boxed( + vec![ + name_column, + column(Justify::Right, Style::PLAIN), + column(Justify::Right, Style::PLAIN), + ], + SQUARE, + (1, 1), + ); + let visible: Vec<&DiffColumnFacts> = mismatched + .iter() + .copied() + .take(TOP_CHANGED_COLUMNS) + .collect(); + for item in &visible { + let matching: i64 = (rows.joined_count - item.mismatched_count).max(0); + table.add_row(texts(&[ + item.name.clone(), + format!("mismatches={}", grouped(item.mismatched_count)), + format!("match={}", percentage(matching, rows.joined_count)), + ])); + } + if mismatched.len() > visible.len() { + table.add_section(); + table.add_row(vec![ + Node::styled("...", MUTED), + Node::styled( + format!( + "and {} more", + grouped((mismatched.len() - visible.len()) as i64) + ), + MUTED, + ), + Node::styled("", MUTED), + ]); + } + if context.text.verbose { + return Node::Table(table); + } + let names: Vec = visible.iter().map(|item| item.name.clone()).collect(); + match examples(model, context, Some(&names)) { + Some(content) => Node::Group(vec![ + Node::Table(table), + Node::plain(""), + content, + Node::plain(""), + Node::styled("Use --verbose to show more example row changes.", MUTED), + ]), + None => Node::Table(table), + } +} + +fn key_label(values: &[DiffNamedValue], separator: &str) -> String { + values + .iter() + .map(|(name, value)| format!("{name}={}", render_key_value(&value.text))) + .collect::>() + .join(separator) +} + +fn examples( + model: &DiffModelFacts, + context: &SectionContext<'_>, + visible: Option<&[String]>, +) -> Option { + let mut grouped_examples: BTreeMap<&str, Vec> = BTreeMap::new(); + for sample in &model.unequal_samples { + let key: String = key_label(&sample.key_values, ", "); + for cell in &sample.changed_cells { + let (left, right) = + render_example_pair(&cell.left.text, &cell.right.text, context.examples); + let difference: String = match left.first_difference { + Some(index) if left.truncated || right.truncated => { + format!( + " (first difference at character {})", + grouped(index as i64 + 1) + ) + } + _ => String::new(), + }; + grouped_examples + .entry(cell.name.as_str()) + .or_default() + .push(format!( + "{key} | {} -> {}{difference}", + display_example_value(&left), + display_example_value(&right) + )); + } + } + let limit: usize = context.text.max_column_examples; + let mut blocks: Vec = Vec::new(); + for (name, all) in grouped_examples { + if visible.is_some_and(|names| !names.iter().any(|visible| visible == name)) { + continue; + } + blocks.push(Node::styled(name, OBJECT)); + for example in all.iter().take(limit) { + blocks.push(Node::plain(format!(" - {example}"))); + } + if all.len() > limit { + blocks.push(Node::styled( + format!( + " showing {} of {} examples", + grouped(limit as i64), + grouped(all.len() as i64) + ), + MUTED, + )); + } + blocks.push(Node::plain("")); + } + blocks.pop().map(|_| Node::Group(blocks)) +} + +fn side_only(label: &str, samples: &[Vec], limit: usize) -> Node { + let mut blocks: Vec = samples + .iter() + .take(limit) + .map(|sample| Node::plain(format!(" - {}", key_label(sample, " | ")))) + .collect(); + if samples.len() > limit { + blocks.push(Node::styled( + format!( + " showing {} of {} {label} only rows", + grouped(limit as i64), + grouped(samples.len() as i64) + ), + MUTED, + )); + } + if blocks.is_empty() { + return Node::styled("No side-only samples collected.", ITALIC); + } + Node::Group(blocks) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/shell_words.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/shell_words.rs new file mode 100644 index 0000000000..dfc11f62df --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/shell_words.rs @@ -0,0 +1,16 @@ +//! Python's `shlex.quote`. + +fn is_safe(character: char) -> bool { + character.is_ascii_alphanumeric() || "_@%+=:,./-".contains(character) +} + +/// One shell word, quoted only when it holds characters outside the safe set. +pub(crate) fn quote(word: &str) -> String { + if word.is_empty() { + return "''".to_owned(); + } + if word.chars().all(is_safe) { + return word.to_owned(); + } + format!("'{}'", word.replace('\'', "'\"'\"'")) +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/size_guard.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/size_guard.rs new file mode 100644 index 0000000000..bc3feae768 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/size_guard.rs @@ -0,0 +1,84 @@ +//! Sizes, payloads and replacement commands of the full-diff size guard. + +use sqlbuild_core::json::models::JsonValue; + +use crate::diff_report::_helpers::numbers::grouped; +use crate::diff_report::_helpers::payload::{integer, object, text}; +use crate::diff_report::_helpers::shell_words::quote; +use crate::diff_report::constants::BOUNDED_WINDOW_PLACEHOLDER; +use crate::diff_report::models::{DiffRequestFacts, FullDiffSideFacts}; + +pub(crate) fn side_text(side: &FullDiffSideFacts) -> String { + let limit: String = side + .max_rows + .map_or_else(|| "unlimited".to_owned(), grouped); + let size: String = match (side.row_count, &side.detail) { + (Some(rows), _) => format!("{} rows", grouped(rows)), + (None, Some(detail)) => format!("size unknown: {detail}"), + (None, None) => "size unknown".to_owned(), + }; + format!("{} {size}, limit {limit}", side.target) +} + +pub(crate) fn optional_integer(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, integer) +} + +pub(crate) fn side_payload(side: &FullDiffSideFacts) -> JsonValue { + object(vec![ + ("target", text(&side.target)), + ("relation", text(&side.relation)), + ("row_count", optional_integer(side.row_count)), + ("max_full_rows", optional_integer(side.max_rows)), + ("exceeds_limit", JsonValue::Bool(side.exceeds_limit)), + ( + "detail", + side.detail.as_deref().map_or(JsonValue::Null, text), + ), + ]) +} + +pub(crate) fn scope_arguments(request: &DiffRequestFacts) -> Vec { + let mut arguments: Vec = Vec::new(); + for (flag, values) in [ + ("--select", &request.select), + ("--exclude", &request.exclude), + ("--key", &request.unique_key_override), + ] { + if !values.is_empty() { + arguments.push(flag.to_owned()); + arguments.extend(values.iter().cloned()); + } + } + if request.unkeyed { + arguments.push("--unkeyed".to_owned()); + } + if !request.excluded_columns_override.is_empty() { + arguments.push("--exclude-column".to_owned()); + arguments.extend(request.excluded_columns_override.iter().cloned()); + } + for tolerance in &request.tolerance_overrides { + arguments.extend(["--tolerance".to_owned(), tolerance.clone()]); + } + if let Some(variables) = &request.cli_vars_json { + arguments.extend(["--vars".to_owned(), variables.clone()]); + } + arguments +} + +pub(crate) fn command(prefix: &[String], mode: &[&str], scope: &[String]) -> String { + prefix + .iter() + .map(String::as_str) + .chain(mode.iter().copied()) + .chain(scope.iter().map(String::as_str)) + .map(|part| { + if part == BOUNDED_WINDOW_PLACEHOLDER { + part.to_owned() + } else { + quote(part) + } + }) + .collect::>() + .join(" ") +} diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs new file mode 100644 index 0000000000..6c87a93a03 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs @@ -0,0 +1,179 @@ +//! Terminal cell widths and Rich's word wrapping of one line of text. + +/// The terminal cells one character occupies: zero for combining marks, two for wide characters. +pub(crate) fn char_cells(character: char) -> usize { + let code: u32 = u32::from(character); + let zero_width: bool = matches!( + code, + 0x0300..=0x036F | 0x200B..=0x200F | 0x20D0..=0x20FF | 0xFE00..=0xFE0F | 0xFE20..=0xFE2F + ); + let wide: bool = matches!( + code, + 0x1100..=0x115F + | 0x2E80..=0x303E + | 0x3041..=0x33FF + | 0x3400..=0x4DBF + | 0x4E00..=0x9FFF + | 0xA000..=0xA4CF + | 0xAC00..=0xD7A3 + | 0xF900..=0xFAFF + | 0xFE30..=0xFE4F + | 0xFF00..=0xFF60 + | 0xFFE0..=0xFFE6 + | 0x1F300..=0x1F64F + | 0x1F900..=0x1F9FF + | 0x20000..=0x3FFFD + ); + if zero_width { + 0 + } else if wide { + 2 + } else { + 1 + } +} + +/// The terminal cells a string occupies. +pub(crate) fn cell_len(text: &str) -> usize { + text.chars().map(char_cells).sum() +} + +/// The longest prefix of `text` that fits `width` cells, padded with spaces to exactly `width`. +pub(crate) fn set_cell_size(text: &str, width: usize) -> String { + let mut result: String = String::new(); + let mut used: usize = 0; + for character in text.chars() { + let cells: usize = char_cells(character); + if used + cells > width { + break; + } + used += cells; + result.push(character); + } + result.push_str(&" ".repeat(width - used)); + result +} + +/// Whitespace-then-word-then-whitespace pieces, as `\s*\S+\s*` matches them. +fn words(characters: &[char]) -> Vec<(usize, usize)> { + let mut pieces: Vec<(usize, usize)> = Vec::new(); + let mut position: usize = 0; + while position < characters.len() { + let start: usize = position; + let mut cursor: usize = position; + while cursor < characters.len() && characters[cursor].is_whitespace() { + cursor += 1; + } + if cursor == characters.len() { + break; + } + while cursor < characters.len() && !characters[cursor].is_whitespace() { + cursor += 1; + } + while cursor < characters.len() && characters[cursor].is_whitespace() { + cursor += 1; + } + pieces.push((start, cursor)); + position = cursor; + } + pieces +} + +fn text_cells(characters: &[char]) -> usize { + characters.iter().copied().map(char_cells).sum() +} + +fn trimmed_end(characters: &[char]) -> &[char] { + let mut end: usize = characters.len(); + while end > 0 && characters[end - 1].is_whitespace() { + end -= 1; + } + &characters[..end] +} + +/// `chop_cells`: consecutive pieces of at most `width` cells. +fn chop_cells(characters: &[char], width: usize) -> Vec> { + let mut lines: Vec> = Vec::new(); + let mut line: Vec = Vec::new(); + let mut size: usize = 0; + for character in characters { + let cells: usize = char_cells(*character); + if size + cells > width && !line.is_empty() { + lines.push(std::mem::take(&mut line)); + size = 0; + } + size += cells; + line.push(*character); + } + if !line.is_empty() { + lines.push(line); + } + lines +} + +/// Rich's `divide_line`: character offsets where a line breaks to fit `width` cells. +fn divide_line(characters: &[char], width: usize, fold: bool) -> Vec { + let mut breaks: Vec = Vec::new(); + let mut offset: usize = 0; + for (word_start, word_end) in words(characters) { + let mut start: usize = word_start; + let word: &[char] = &characters[word_start..word_end]; + let word_length: usize = text_cells(trimmed_end(word)); + if width.saturating_sub(offset) >= word_length && offset <= width { + offset += text_cells(word); + } else if word_length > width { + if fold { + let folded: Vec> = chop_cells(word, width); + let count: usize = folded.len(); + for (index, piece) in folded.iter().enumerate() { + if start > 0 { + breaks.push(start); + } + if index + 1 == count { + offset = text_cells(piece); + } else { + start += piece.len(); + } + } + } else { + if start > 0 { + breaks.push(start); + } + offset = text_cells(word); + } + } else if offset > 0 && start > 0 { + breaks.push(start); + offset = text_cells(word); + } + } + breaks +} + +/// One source line wrapped to `width` cells like `Text.wrap` without justification. +pub(crate) fn wrap_line(line: &str, width: usize, fold: bool) -> Vec { + let characters: Vec = line.chars().collect(); + let mut pieces: Vec = Vec::new(); + let mut previous: usize = 0; + let mut breaks: Vec = divide_line(&characters, width, fold); + breaks.push(characters.len()); + for offset in breaks { + let mut piece: Vec = characters[previous..offset].to_vec(); + previous = offset; + if piece.len() > width { + let excess: usize = piece.len() - width; + let trailing: usize = piece.len() - trimmed_end(&piece).len(); + piece.truncate(piece.len() - trailing.min(excess)); + } + let text: String = piece.into_iter().collect(); + pieces.push(if cell_len(&text) > width { + if fold { + set_cell_size(&text, width) + } else { + set_cell_size(&text, width.saturating_sub(1)) + "…" + } + } else { + text + }); + } + pieces +} diff --git a/crates/sqlbuild-cli/src/diff_report/constants.rs b/crates/sqlbuild-cli/src/diff_report/constants.rs new file mode 100644 index 0000000000..bda83c40ea --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/constants.rs @@ -0,0 +1,42 @@ +//! Fixed widths and limits of the diff summary. + +pub(crate) const SUMMARY_WIDTH: usize = 110; +pub(crate) const TOP_CHANGED_COLUMNS: usize = 5; +pub(crate) const MAX_RENDERED_KEY_LENGTH: usize = 256; +pub(crate) const CHANGED_COLUMN_NAME_WIDTH: usize = 32; +pub(crate) const QUERY_INPUT_KIND: &str = "query"; +pub(crate) const FINDINGS_OUTCOME: &str = "findings"; +pub(crate) const PASS_OUTCOME: &str = "pass"; +pub(crate) const INCOMPLETE_OUTCOME: &str = "incomplete"; +pub(crate) const BOUNDED_WINDOW_PLACEHOLDER: &str = ""; +/// Root expression kinds a raw diff query may have. +pub(crate) const READ_ONLY_QUERY_ROOT_KEYS: [&str; 5] = + ["except", "intersect", "select", "union", "values"]; +/// Expression kinds that make a raw diff query change data or state. +pub(crate) const MUTATING_QUERY_KEYS: [&str; 19] = [ + "alter", + "command", + "copy", + "create", + "delete", + "drop", + "execute", + "grant", + "insert", + "load_data", + "merge", + "pragma", + "revoke", + "set", + "transaction", + "truncate", + "uncache", + "update", + "use", +]; +pub(crate) const INCOMPLETE_PREPARATION_CODES: [&str; 9] = [ + "C229", "C230", "C231", "C232", "C233", "C234", "C235", "C240", "C244", +]; +pub(crate) const INCOMPLETE_EXECUTION_CODES: [&str; 3] = ["C237", "C238", "C239"]; +pub(crate) const DEFAULT_EXAMPLES: i64 = 3; +pub(crate) const VERBOSE_EXAMPLES: i64 = 10; diff --git a/crates/sqlbuild-cli/src/diff_report/errors.rs b/crates/sqlbuild-cli/src/diff_report/errors.rs new file mode 100644 index 0000000000..c0c8c8b9b5 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/errors.rs @@ -0,0 +1,9 @@ +//! Rejections of `sqb diff` requests and inputs. + +/// A rejected diff request or input: message, `C` code and optional help. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DiffUserError { + pub message: String, + pub code: String, + pub help: Option, +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/diff_exit_code.rs b/crates/sqlbuild-cli/src/diff_report/main/diff_exit_code.rs new file mode 100644 index 0000000000..e980677d3b --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/diff_exit_code.rs @@ -0,0 +1,22 @@ +//! The outcome and exit code of a finished diff. + +use crate::diff_report::_helpers::findings::{has_findings, has_schema_differences}; +use crate::diff_report::constants::{FINDINGS_OUTCOME, INCOMPLETE_OUTCOME, PASS_OUTCOME}; +use crate::diff_report::models::DiffModelFacts; + +/// `pass` (0), `findings` (1), or `incomplete` (2) when a query schema blocked row comparison. +#[must_use] +pub fn diff_exit_code(models: &[DiffModelFacts], query_row_comparison: bool) -> (String, i32) { + let incomplete: bool = query_row_comparison + && models + .iter() + .any(|model| model.rows.is_none() && has_schema_differences(model)); + let (outcome, code) = if incomplete { + (INCOMPLETE_OUTCOME, 2) + } else if has_findings(models) { + (FINDINGS_OUTCOME, 1) + } else { + (PASS_OUTCOME, 0) + }; + (outcome.to_owned(), code) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/effective_max_examples.rs b/crates/sqlbuild-cli/src/diff_report/main/effective_max_examples.rs new file mode 100644 index 0000000000..fe916f2b34 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/effective_max_examples.rs @@ -0,0 +1,13 @@ +//! How many examples a diff shows per column or side when not set explicitly. + +use crate::diff_report::constants::{DEFAULT_EXAMPLES, VERBOSE_EXAMPLES}; + +/// The explicit limit, else ten when verbose and three otherwise. +#[must_use] +pub fn effective_max_examples(explicit: Option, verbose: bool) -> i64 { + explicit.unwrap_or(if verbose { + VERBOSE_EXAMPLES + } else { + DEFAULT_EXAMPLES + }) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/is_incomplete_query_failure.rs b/crates/sqlbuild-cli/src/diff_report/main/is_incomplete_query_failure.rs new file mode 100644 index 0000000000..c63c7fb723 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/is_incomplete_query_failure.rs @@ -0,0 +1,13 @@ +//! Whether a raw-query diff user error means incomplete evidence rather than a failed run. + +use crate::diff_report::constants::{INCOMPLETE_EXECUTION_CODES, INCOMPLETE_PREPARATION_CODES}; +use crate::diff_report::types::QueryDiffPhase; + +/// Input and preflight rejections are incomplete (exit 2); other failures are execution failures. +#[must_use] +pub fn is_incomplete_query_failure(phase: QueryDiffPhase, code: &str) -> bool { + match phase { + QueryDiffPhase::Preparation => INCOMPLETE_PREPARATION_CODES.contains(&code), + QueryDiffPhase::Execution => INCOMPLETE_EXECUTION_CODES.contains(&code), + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/is_query_diff_request.rs b/crates/sqlbuild-cli/src/diff_report/main/is_query_diff_request.rs new file mode 100644 index 0000000000..43bb8fcec5 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/is_query_diff_request.rs @@ -0,0 +1,9 @@ +//! Whether a diff compares raw queries instead of project models. + +use crate::diff_report::models::DiffRequestFacts; + +/// Any raw-query input was supplied. +#[must_use] +pub fn is_query_diff_request(request: &DiffRequestFacts) -> bool { + request.left_query || request.left_query_file || request.right_query || request.right_query_file +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/mod.rs b/crates/sqlbuild-cli/src/diff_report/main/mod.rs new file mode 100644 index 0000000000..4535034e41 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/mod.rs @@ -0,0 +1,12 @@ +pub mod diff_exit_code; +pub mod effective_max_examples; +pub mod is_incomplete_query_failure; +pub mod is_query_diff_request; +pub mod parse_diff_name_range; +pub mod query_column_problem; +pub mod read_only_query_problem; +pub mod render_diff_error_json; +pub mod render_diff_json; +pub mod render_diff_text; +pub mod size_guard_report; +pub mod validate_diff_request; diff --git a/crates/sqlbuild-cli/src/diff_report/main/parse_diff_name_range.rs b/crates/sqlbuild-cli/src/diff_report/main/parse_diff_name_range.rs new file mode 100644 index 0000000000..791703b50a --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/parse_diff_name_range.rs @@ -0,0 +1,18 @@ +//! The positional `FROM[:TO]` target range of `sqb diff`. + +use crate::diff_report::_helpers::request_rules::user_error; +use crate::diff_report::errors::DiffUserError; + +/// `FROM:TO` or a bare `FROM`, which leaves `TO` unset. +pub fn parse_diff_name_range(range: &str) -> Result<(String, Option), DiffUserError> { + let (from_name, to_name): (&str, Option<&str>) = match range.split_once(':') { + Some((from_name, to_name)) => (from_name, Some(to_name)), + None => (range, None), + }; + if from_name.is_empty() + || to_name.is_some_and(|to_name| to_name.is_empty() || to_name.contains(':')) + { + return Err(user_error("diff range must be FROM:TO or FROM", "C209")); + } + Ok((from_name.to_owned(), to_name.map(str::to_owned))) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/query_column_problem.rs b/crates/sqlbuild-cli/src/diff_report/main/query_column_problem.rs new file mode 100644 index 0000000000..dd2f1be65a --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/query_column_problem.rs @@ -0,0 +1,56 @@ +//! Preflight checks of the columns one raw diff query returns. + +use crate::diff_report::errors::DiffUserError; + +/// No columns (C237), duplicate names ignoring case (C238), or too many columns (C239). +pub fn query_column_problem( + label: &str, + columns: &[String], + max_columns: Option, +) -> Result<(), DiffUserError> { + let display: String = if label.to_lowercase().ends_with("query") { + label.to_owned() + } else { + format!("{label} query") + }; + if columns.is_empty() { + return Err(DiffUserError { + message: format!("{display} returned no columns"), + code: "C237".to_owned(), + help: None, + }); + } + let lowered: Vec = columns.iter().map(|column| column.to_lowercase()).collect(); + let mut sorted: Vec<&String> = lowered.iter().collect(); + sorted.sort(); + let mut duplicates: Vec<&String> = Vec::new(); + for pair in sorted.windows(2) { + if pair[0] == pair[1] { + duplicates.push(pair[0]); + } + } + duplicates.sort(); + duplicates.dedup(); + if !duplicates.is_empty() { + let names: Vec<&str> = duplicates.into_iter().map(String::as_str).collect(); + return Err(DiffUserError { + message: format!( + "{display} returns duplicate column names: {}", + names.join(", ") + ), + code: "C238".to_owned(), + help: Some("add unique aliases before comparing query results".to_owned()), + }); + } + match max_columns { + Some(limit) if columns.len() as i64 > limit => Err(DiffUserError { + message: format!( + "{display} has {} columns, exceeding --max-columns {limit}", + columns.len() + ), + code: "C239".to_owned(), + help: None, + }), + _ => Ok(()), + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/read_only_query_problem.rs b/crates/sqlbuild-cli/src/diff_report/main/read_only_query_problem.rs new file mode 100644 index 0000000000..f187afc9aa --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/read_only_query_problem.rs @@ -0,0 +1,33 @@ +//! Whether one parsed raw diff query is a single read-only expression. + +use crate::diff_report::_helpers::request_rules::user_error; +use crate::diff_report::constants::{MUTATING_QUERY_KEYS, READ_ONLY_QUERY_ROOT_KEYS}; +use crate::diff_report::errors::DiffUserError; +use crate::diff_report::models::QueryShapeFacts; + +/// A non-query root (C235), any data-changing node (C240), or an `INTO`/lock clause (C244). +pub fn read_only_query_problem(label: &str, shape: &QueryShapeFacts) -> Result<(), DiffUserError> { + if !READ_ONLY_QUERY_ROOT_KEYS.contains(&shape.root_key.as_str()) { + return Err(user_error( + &format!("{label} query must be one read-only SELECT, WITH, or VALUES expression"), + "C235", + )); + } + if shape + .node_keys + .iter() + .any(|key| MUTATING_QUERY_KEYS.contains(&key.as_str())) + { + return Err(user_error( + &format!("{label} query must not contain data-changing or administrative statements"), + "C240", + )); + } + if shape.writes_or_locks { + return Err(user_error( + &format!("{label} query must not write a relation or acquire row locks"), + "C244", + )); + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/render_diff_error_json.rs b/crates/sqlbuild-cli/src/diff_report/main/render_diff_error_json.rs new file mode 100644 index 0000000000..1280a07b75 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/render_diff_error_json.rs @@ -0,0 +1,33 @@ +//! The machine-readable document of an incomplete or failed diff. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::diff_report::_helpers::payload::{integer, object, text}; +use crate::diff_report::models::DiffErrorFacts; + +/// The error document text, without a trailing newline. +pub fn render_diff_error_json(facts: &DiffErrorFacts) -> Result { + let mut error: Vec<(&str, JsonValue)> = vec![ + ("code", text(&facts.code)), + ("message", text(&facts.message)), + ]; + if let Some(help) = &facts.help { + error.push(("help", text(help))); + } + let mut entries: Vec<(&str, JsonValue)> = vec![ + ("schema_version", integer(1)), + ("status", text(&facts.status)), + ("outcome", text(&facts.status)), + ("error", object(error)), + ("models", JsonValue::Array(Vec::new())), + ("query_comparisons", JsonValue::Array(Vec::new())), + ]; + if let Some(size_guard) = &facts.size_guard { + entries.push(("size_guard", size_guard.clone())); + } + let mut settings: StdlibJsonOptions = StdlibJsonOptions::new(Some(2)); + settings.sort_keys = true; + dumps(&object(entries), &JsonDialect::Stdlib(settings)) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/render_diff_json.rs b/crates/sqlbuild-cli/src/diff_report/main/render_diff_json.rs new file mode 100644 index 0000000000..84e56dd2b3 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/render_diff_json.rs @@ -0,0 +1,19 @@ +//! The structured diff document, serialized as `json.dumps(indent=2, sort_keys=True)`. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; + +use crate::diff_report::_helpers::payload::document; +use crate::diff_report::models::{DiffDocumentOptions, DiffModelFacts}; + +/// The diff document text, without a trailing newline. +pub fn render_diff_json( + models: &[DiffModelFacts], + options: &DiffDocumentOptions, +) -> Result { + let mut settings: StdlibJsonOptions = StdlibJsonOptions::new(Some(2)); + settings.sort_keys = true; + settings.allow_nan = false; + dumps(&document(models, options), &JsonDialect::Stdlib(settings)) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/render_diff_text.rs b/crates/sqlbuild-cli/src/diff_report/main/render_diff_text.rs new file mode 100644 index 0000000000..43a4480e2e --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/render_diff_text.rs @@ -0,0 +1,63 @@ +//! The terminal diff summary. + +use crate::diff_report::_helpers::findings::resolved_outcome; +use crate::diff_report::_helpers::layout::{Node, print_nodes}; +use crate::diff_report::_helpers::numbers::grouped; +use crate::diff_report::_helpers::sections::{BOLD, MUTED, OBJECT, SectionContext, model_nodes}; +use crate::diff_report::constants::{QUERY_INPUT_KIND, SUMMARY_WIDTH}; +use crate::diff_report::models::{DiffDocumentOptions, DiffModelFacts, DiffTextOptions}; + +/// Render the summary panel, then every model's overview and sections. +#[must_use] +pub fn render_diff_text( + models: &[DiffModelFacts], + document: &DiffDocumentOptions, + options: &DiffTextOptions, +) -> String { + if models.is_empty() { + return print_nodes( + &[Node::plain("No models selected for diff.")], + SUMMARY_WIDTH, + options.use_color, + ); + } + let input_label: &str = if models + .iter() + .all(|model| model.input_kind == QUERY_INPUT_KIND) + { + "query pairs" + } else { + "models" + }; + let mut nodes: Vec = vec![ + Node::Panel("SQLBuild Diff Summary".to_owned(), BOLD), + Node::styled( + format!("{} vs {}", document.from_label, document.to_label), + OBJECT, + ), + Node::plain(format!( + "selected {input_label}: {}", + grouped(models.len() as i64) + )), + Node::plain(format!( + "outcome: {}", + resolved_outcome(models, document.outcome.as_deref()) + )), + Node::plain(""), + ]; + let context: SectionContext<'_> = SectionContext { + from_label: &document.from_label, + to_label: &document.to_label, + text: options, + examples: document.examples, + }; + for (index, model) in models.iter().enumerate() { + if index > 0 { + nodes.push(Node::plain("")); + nodes.push(Node::styled("─".repeat(SUMMARY_WIDTH), MUTED)); + nodes.push(Node::plain("")); + } + nodes.extend(model_nodes(model, &context)); + } + print_nodes(&nodes, SUMMARY_WIDTH, options.use_color) +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/size_guard_report.rs b/crates/sqlbuild-cli/src/diff_report/main/size_guard_report.rs new file mode 100644 index 0000000000..23427cb684 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/size_guard_report.rs @@ -0,0 +1,94 @@ +//! The C270 report of a default full diff stopped by table sizes. + +use sqlbuild_core::json::models::JsonValue; + +use crate::diff_report::_helpers::payload::{object, text}; +use crate::diff_report::_helpers::size_guard::{command, scope_arguments, side_payload, side_text}; +use crate::diff_report::constants::BOUNDED_WINDOW_PLACEHOLDER; +use crate::diff_report::models::{DiffRequestFacts, FullDiffModelFacts, SizeGuardReport}; + +/// Every blocked model, the explicit replacement commands, and how to raise the limits. +#[must_use] +pub fn size_guard_report( + request: &DiffRequestFacts, + (from_target, to_target): (&str, &str), + guard_message: &str, + blocked: &[FullDiffModelFacts], +) -> SizeGuardReport { + let scope: Vec = scope_arguments(request); + let mut prefix: Vec = vec!["sqb".to_owned()]; + if let Some(project_dir) = &request.project_dir { + prefix.extend(["--project-dir".to_owned(), project_dir.clone()]); + } + prefix.extend(["diff".to_owned(), format!("{from_target}:{to_target}")]); + let full: String = command(&prefix, &["--full"], &scope); + let bounded: Option = blocked + .iter() + .all(|model| model.has_cursor) + .then(|| command(&prefix, &["--bounded", BOUNDED_WINDOW_PLACEHOLDER], &scope)); + let schema_only: String = command(&prefix, &["--schema-only"], &scope); + let mut lines: Vec = vec![guard_message.to_owned()]; + for model in blocked { + lines.push(format!( + " {}: {}; {}{}", + model.name, + side_text(&model.left), + side_text(&model.right), + if model.has_cursor { "" } else { "; no cursor" } + )); + } + lines.push("Run one of these instead:".to_owned()); + lines.push(format!(" {full}")); + if let Some(bounded) = &bounded { + lines.push(format!(" {bounded}")); + } + lines.push(format!(" {schema_only}")); + let mut over_targets: Vec<&str> = Vec::new(); + for side in blocked.iter().flat_map(|model| [&model.left, &model.right]) { + if side.exceeds_limit && !over_targets.contains(&side.target.as_str()) { + over_targets.push(&side.target); + } + } + let bounded_help: String = if bounded.is_some() { + format!( + ", --bounded compares a recent cursor window (replace {BOUNDED_WINDOW_PLACEHOLDER} with a duration such as 7d)" + ) + } else { + String::new() + }; + let raise: Vec = over_targets + .iter() + .map(|target| format!("targets.{target}.diff.max_full_rows")) + .collect(); + let models: Vec = blocked + .iter() + .map(|model| { + object(vec![ + ("name", text(&model.name)), + ("has_cursor", JsonValue::Bool(model.has_cursor)), + ("from", side_payload(&model.left)), + ("to", side_payload(&model.right)), + ]) + }) + .collect(); + SizeGuardReport { + message: lines.join("\n"), + help: format!( + "--full compares every row{bounded_help}, and --schema-only compares columns only; or raise {}", + raise.join(" and ") + ), + details: object(vec![ + ("from_target", text(from_target)), + ("to_target", text(to_target)), + ("models", JsonValue::Array(models)), + ( + "commands", + object(vec![ + ("full", text(&full)), + ("bounded", bounded.as_deref().map_or(JsonValue::Null, text)), + ("schema_only", text(&schema_only)), + ]), + ), + ]), + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/main/validate_diff_request.rs b/crates/sqlbuild-cli/src/diff_report/main/validate_diff_request.rs new file mode 100644 index 0000000000..37e1eb05d7 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/main/validate_diff_request.rs @@ -0,0 +1,20 @@ +//! Reject contradictory or out-of-range `sqb diff` flags before any discovery. + +use crate::diff_report::_helpers::request_rules::{ + common_violation, model_violation, query_violation, user_error, +}; +use crate::diff_report::errors::DiffUserError; +use crate::diff_report::main::is_query_diff_request::is_query_diff_request; +use crate::diff_report::models::DiffRequestFacts; + +/// The first violated rule, checked in the order users see them reported. +pub fn validate_diff_request(request: &DiffRequestFacts) -> Result<(), DiffUserError> { + let violation: Option<(&str, &str)> = common_violation(request).or_else(|| { + if is_query_diff_request(request) { + query_violation(request) + } else { + model_violation(request) + } + }); + violation.map_or(Ok(()), |(message, code)| Err(user_error(message, code))) +} diff --git a/crates/sqlbuild-cli/src/diff_report/mod.rs b/crates/sqlbuild-cli/src/diff_report/mod.rs new file mode 100644 index 0000000000..50aec51473 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/mod.rs @@ -0,0 +1,11 @@ +//! The `sqb diff` summary and JSON documents rendered from plain diff facts. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cli/src/diff_report/models.rs b/crates/sqlbuild-cli/src/diff_report/models.rs new file mode 100644 index 0000000000..50ed8eb30c --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/models.rs @@ -0,0 +1,205 @@ +//! Plain facts of one diff result and the options that render it. + +use sqlbuild_core::json::models::JsonValue; + +use crate::diff_report::types::{DiffNamedValue, DiffTypeChange}; + +/// One fetched warehouse value: its Python `str()` text and its JSON document form. +#[derive(Clone, Debug, PartialEq)] +pub struct DiffValue { + pub text: String, + pub json: JsonValue, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffColumnFacts { + pub name: String, + pub mismatched_count: i64, + /// `str()` of the absolute and relative tolerance, when one is configured. + pub tolerance: Option<(Option, Option)>, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffCoverageFacts { + pub row_count: i64, + pub minimum_cursor: Option, + pub maximum_cursor: Option, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffRowFacts { + pub left_count: i64, + pub right_count: i64, + pub joined_count: i64, + pub equal_count: i64, + pub unequal_count: i64, + pub left_only_count: i64, + pub right_only_count: i64, + pub columns: Vec, + pub population_count: i64, + pub compared_count: i64, + /// The configured sample row limit and seed. + pub sampling: Option<(i64, i64)>, + pub left_coverage: Option, + pub right_coverage: Option, + pub cursor_column: Option, + pub cursor_start: Option, + pub cursor_end: Option, + pub exhaustive: bool, + /// Whether both sides report different cursor extents, by Python value equality. + pub coverage_differs: bool, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct DiffChangedCell { + pub name: String, + pub left: DiffValue, + pub right: DiffValue, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffSampleRow { + pub key_values: Vec, + pub changed_cells: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffModelFacts { + pub name: String, + pub left_relation: String, + pub right_relation: String, + pub added_columns: Vec, + pub removed_columns: Vec, + pub type_changes: Vec, + pub unique_key: Vec, + pub rows: Option, + pub unequal_samples: Vec, + pub left_only_keys: Vec>, + pub right_only_keys: Vec>, + pub bounded_fallback: bool, + pub excluded_columns: Vec, + pub input_kind: String, + pub unkeyed: bool, +} + +/// How example values render: an optional per-value length bound, or suppressed entirely. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DiffExampleOptions { + pub max_value_length: Option, + pub suppress_values: bool, +} + +/// Everything a rendered diff document shows besides the per-model facts. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffDocumentOptions { + pub from_label: String, + pub to_label: String, + pub examples: DiffExampleOptions, + /// An explicit outcome, such as `incomplete`; otherwise derived from the findings. + pub outcome: Option, + /// Per-phase durations of a query diff, in seconds. + pub phase_seconds: Option>, +} + +/// The terminal summary's presentation options. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DiffTextOptions { + pub mode_label: String, + pub use_color: bool, + pub verbose: bool, + pub max_column_examples: usize, + pub max_row_only_examples: usize, +} + +/// A failed or incomplete diff reported as JSON. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiffErrorFacts { + pub status: String, + pub code: String, + pub message: String, + pub help: Option, + pub size_guard: Option, +} + +/// One value rendered for display: bounded around the first difference, or suppressed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct RenderedValue { + pub(crate) text: Option, + pub(crate) original_length: usize, + pub(crate) truncated: bool, + pub(crate) suppressed: bool, + pub(crate) first_difference: Option, +} + +/// The `sqb diff` flags that request validation reads. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DiffRequestFacts { + pub from_name: Option, + pub to_name: Option, + pub full: bool, + pub schema_only: bool, + pub bounded: Option, + pub max_column_examples: Option, + pub max_row_only_examples: Option, + pub sample_rows: Option, + pub sample_seed: Option, + pub exhaustive: bool, + pub max_models: Option, + pub max_columns: Option, + pub max_value_length: Option, + pub full_example_values: bool, + pub suppress_example_values: bool, + pub select: Vec, + pub exclude: Vec, + pub selected_target: Option, + pub left_query: bool, + pub left_query_file: bool, + pub right_query: bool, + pub right_query_file: bool, + pub left_label: Option, + pub right_label: Option, + pub unique_key_override: Vec, + pub unkeyed: bool, + pub excluded_columns_override: Vec, + pub tolerance_overrides: Vec, + /// `json.dumps(cli_vars, sort_keys=True)` when variables were passed. + pub cli_vars_json: Option, + pub project_dir: Option, +} + +/// One side of a model the default full diff refused to compare. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FullDiffSideFacts { + pub target: String, + pub relation: String, + pub row_count: Option, + pub max_rows: Option, + pub exceeds_limit: bool, + pub detail: Option, +} + +/// One model blocked by the default full diff size guard. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FullDiffModelFacts { + pub name: String, + pub has_cursor: bool, + pub left: FullDiffSideFacts, + pub right: FullDiffSideFacts, +} + +/// The C270 size-guard error: message, help and machine-readable details. +#[derive(Clone, Debug, PartialEq)] +pub struct SizeGuardReport { + pub message: String, + pub help: String, + pub details: JsonValue, +} + +/// The parsed shape of one raw diff query: expression kinds, lowercased, and write clauses. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct QueryShapeFacts { + pub root_key: String, + pub node_keys: Vec, + /// The root expression has an `INTO` target or row-locking clauses. + pub writes_or_locks: bool, +} diff --git a/crates/sqlbuild-cli/src/diff_report/tests.rs b/crates/sqlbuild-cli/src/diff_report/tests.rs new file mode 100644 index 0000000000..eed637fb30 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_command.rs"] +mod command; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_render.rs"] +mod render; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/src/diff_report/tests/helpers.rs b/crates/sqlbuild-cli/src/diff_report/tests/helpers.rs new file mode 100644 index 0000000000..150fe5f99f --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/tests/helpers.rs @@ -0,0 +1,97 @@ +use sqlbuild_core::json::models::JsonValue; + +use crate::diff_report::models::{ + DiffColumnFacts, DiffDocumentOptions, DiffExampleOptions, DiffModelFacts, DiffRequestFacts, + DiffRowFacts, DiffValue, FullDiffModelFacts, FullDiffSideFacts, +}; + +pub(crate) fn changed_rows(unequal: i64) -> DiffRowFacts { + DiffRowFacts { + left_count: 3, + right_count: 3, + joined_count: 3, + equal_count: 3 - unequal, + unequal_count: unequal, + columns: vec![DiffColumnFacts { + name: "amount".to_owned(), + mismatched_count: unequal, + tolerance: None, + }], + population_count: 3, + compared_count: 3, + exhaustive: true, + ..DiffRowFacts::default() + } +} + +pub(crate) fn model(rows: Option, added_columns: &[&str]) -> DiffModelFacts { + DiffModelFacts { + name: "orders".to_owned(), + left_relation: "prod.orders".to_owned(), + right_relation: "dev.orders".to_owned(), + added_columns: added_columns + .iter() + .map(|name| (*name).to_owned()) + .collect(), + unique_key: vec!["order_id".to_owned()], + rows, + input_kind: "model".to_owned(), + ..DiffModelFacts::default() + } +} + +pub(crate) fn document() -> DiffDocumentOptions { + DiffDocumentOptions { + from_label: "prod".to_owned(), + to_label: "dev".to_owned(), + examples: DiffExampleOptions { + max_value_length: Some(160), + suppress_values: false, + }, + ..DiffDocumentOptions::default() + } +} + +pub(crate) fn cursor_rows() -> DiffRowFacts { + DiffRowFacts { + cursor_column: Some("ts".to_owned()), + cursor_start: Some(DiffValue { + text: "nan".to_owned(), + json: JsonValue::Float(f64::NAN), + }), + ..changed_rows(1) + } +} + +pub(crate) fn request(build: impl FnOnce(&mut DiffRequestFacts)) -> DiffRequestFacts { + let mut facts: DiffRequestFacts = DiffRequestFacts::default(); + build(&mut facts); + facts +} + +pub(crate) fn blocked_orders(has_cursor: bool) -> Vec { + vec![FullDiffModelFacts { + name: "orders".to_owned(), + has_cursor, + left: FullDiffSideFacts { + target: "prod".to_owned(), + relation: "prod.orders".to_owned(), + row_count: Some(12_500_000), + max_rows: Some(10_000_000), + exceeds_limit: true, + detail: None, + }, + right: FullDiffSideFacts { + target: "dev".to_owned(), + relation: "dev.orders".to_owned(), + row_count: None, + max_rows: Some(10_000_000), + exceeds_limit: true, + detail: Some("no row count (view)".to_owned()), + }, + }] +} + +pub(crate) fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} diff --git a/crates/sqlbuild-cli/src/diff_report/tests/test_command.rs b/crates/sqlbuild-cli/src/diff_report/tests/test_command.rs new file mode 100644 index 0000000000..ed284add8f --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/tests/test_command.rs @@ -0,0 +1,250 @@ +use crate::diff_report::main::parse_diff_name_range::parse_diff_name_range; +use crate::diff_report::main::query_column_problem::query_column_problem; +use crate::diff_report::main::size_guard_report::size_guard_report; +use crate::diff_report::main::validate_diff_request::validate_diff_request; +use crate::diff_report::tests::helpers::{blocked_orders, request, strings}; +use crate::diff_report::tests::test_types::{ + ColumnsTestCase, NameRangeTestCase, RequestTestCase, SizeGuardTestCase, +}; + +#[test] +fn given_target_range_when_parsing_then_targets_or_code_match() { + let parsed = |from: &str, to: Option<&str>| Ok((from.to_owned(), to.map(str::to_owned))); + let test_cases = [ + NameRangeTestCase { + description: "explicit range keeps both targets", + range: "prod:dev", + expected_targets: parsed("prod", Some("dev")), + }, + NameRangeTestCase { + description: "bare target leaves TO unset", + range: "prod", + expected_targets: parsed("prod", None), + }, + NameRangeTestCase { + description: "empty target", + range: "", + expected_targets: Err("C209".to_owned()), + }, + NameRangeTestCase { + description: "empty TO", + range: "prod:", + expected_targets: Err("C209".to_owned()), + }, + NameRangeTestCase { + description: "empty FROM", + range: ":dev", + expected_targets: Err("C209".to_owned()), + }, + NameRangeTestCase { + description: "too many separators", + range: "a:b:c", + expected_targets: Err("C209".to_owned()), + }, + ]; + + for test_case in test_cases { + let parsed = parse_diff_name_range(test_case.range).map_err(|error| error.code); + + assert_eq!( + parsed, test_case.expected_targets, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_request_flags_when_validating_then_first_violation_is_reported() { + let test_cases = [ + RequestTestCase { + description: "model diff with a range and selector passes", + request: request(|facts| facts.from_name = Some("prod".to_owned())), + expected_code: None, + }, + RequestTestCase { + description: "two comparison modes", + request: request(|facts| { + facts.full = true; + facts.schema_only = true; + }), + expected_code: Some("C201"), + }, + RequestTestCase { + description: "duplicate keys ignoring case", + request: request(|facts| { + facts.from_name = Some("prod".to_owned()); + facts.unique_key_override = strings(&["ID", "id"]); + }), + expected_code: Some("C242"), + }, + RequestTestCase { + description: "model diff without a range", + request: request(|_| {}), + expected_code: Some("C224"), + }, + RequestTestCase { + description: "raw-query diff without a right query", + request: request(|facts| facts.left_query = true), + expected_code: Some("C218"), + }, + RequestTestCase { + description: "raw-query diff without key or unkeyed", + request: request(|facts| { + facts.left_query = true; + facts.right_query_file = true; + }), + expected_code: Some("C221"), + }, + RequestTestCase { + description: "raw-query diff rejects selectors before inputs", + request: request(|facts| { + facts.left_query = true; + facts.select = strings(&["orders"]); + }), + expected_code: Some("C213"), + }, + ]; + + for test_case in test_cases { + let code = validate_diff_request(&test_case.request) + .err() + .map(|error| error.code); + + assert_eq!( + code.as_deref(), + test_case.expected_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_blocked_models_when_building_guard_report_then_names_sizes_and_commands() { + let test_cases = [ + SizeGuardTestCase { + description: "model with cursor offers bounded command", + request: request(|facts| facts.select = strings(&["orders"])), + has_cursor: true, + expected_full: "sqb diff prod:dev --full --select orders", + expected_bounded: Some("sqb diff prod:dev --bounded --select orders"), + expected_fragments: &[ + "orders: prod 12,500,000 rows, limit 10,000,000; dev size unknown: no row count (view), limit 10,000,000", + "Run one of these instead:", + "sqb diff prod:dev --schema-only --select orders", + ], + }, + SizeGuardTestCase { + description: "model without cursor omits bounded command and quotes key flags", + request: request(|facts| { + facts.select = strings(&["tag:daily", "path:models/marts"]); + facts.unique_key_override = strings(&["order_id", "line number"]); + }), + has_cursor: false, + expected_full: "sqb diff prod:dev --full --select tag:daily path:models/marts --key order_id 'line number'", + expected_bounded: None, + expected_fragments: &["; no cursor"], + }, + SizeGuardTestCase { + description: "project dir is a global flag placed before diff", + request: request(|facts| { + facts.select = strings(&["orders"]); + facts.project_dir = Some("projects/shop".to_owned()); + }), + has_cursor: false, + expected_full: "sqb --project-dir projects/shop diff prod:dev --full --select orders", + expected_bounded: None, + expected_fragments: &[], + }, + ]; + + for test_case in test_cases { + let report = size_guard_report( + &test_case.request, + ("prod", "dev"), + "full diff stopped", + &blocked_orders(test_case.has_cursor), + ); + let lines: Vec<&str> = report.message.lines().map(str::trim).collect(); + let found: Vec<&str> = test_case + .expected_fragments + .iter() + .copied() + .filter(|fragment| report.message.contains(fragment)) + .collect(); + + assert_eq!( + ( + lines.contains(&test_case.expected_full), + test_case + .expected_bounded + .is_none_or(|bounded| lines.contains(&bounded)), + report + .help + .contains("targets.prod.diff.max_full_rows and targets.dev.diff.max_full_rows"), + report.help.contains("--bounded") == test_case.expected_bounded.is_some(), + found, + ), + ( + true, + true, + true, + true, + test_case.expected_fragments.to_vec() + ), + "{}: {}", + test_case.description, + report.message + ); + } +} + +#[test] +fn given_query_columns_when_preflighting_then_problems_are_coded() { + let test_cases = [ + ColumnsTestCase { + description: "distinct columns pass", + columns: &["id", "v"], + max_columns: Some(2), + expected_outcome: Ok(()), + }, + ColumnsTestCase { + description: "no columns", + columns: &[], + max_columns: None, + expected_outcome: Err(("C237", "left query returned no columns")), + }, + ColumnsTestCase { + description: "duplicates ignore case", + columns: &["ID", "id", "b", "B"], + max_columns: None, + expected_outcome: Err(("C238", "left query returns duplicate column names: b, id")), + }, + ColumnsTestCase { + description: "too many columns", + columns: &["a", "b"], + max_columns: Some(1), + expected_outcome: Err(( + "C239", + "left query has 2 columns, exceeding --max-columns 1", + )), + }, + ]; + + for test_case in test_cases { + let outcome = + query_column_problem("left", &strings(test_case.columns), test_case.max_columns); + let outcome = outcome.map_err(|error| (error.code, error.message)); + let outcome = outcome + .as_ref() + .copied() + .map_err(|(code, message)| (code.as_str(), message.as_str())); + + assert_eq!( + outcome, test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/tests/test_render.rs b/crates/sqlbuild-cli/src/diff_report/tests/test_render.rs new file mode 100644 index 0000000000..6f85c27046 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/tests/test_render.rs @@ -0,0 +1,222 @@ +use crate::diff_report::_helpers::evidence::{ + display_example_value, render_example_pair, render_key_value, +}; +use crate::diff_report::main::diff_exit_code::diff_exit_code; +use crate::diff_report::main::render_diff_error_json::render_diff_error_json; +use crate::diff_report::main::render_diff_json::render_diff_json; +use crate::diff_report::main::render_diff_text::render_diff_text; +use crate::diff_report::models::{DiffErrorFacts, DiffExampleOptions, DiffTextOptions}; +use crate::diff_report::tests::helpers::{changed_rows, cursor_rows, document, model}; +use crate::diff_report::tests::test_types::{ + EvidenceTestCase, JsonTestCase, KeyTestCase, OutcomeTestCase, SummaryTestCase, +}; + +#[test] +fn given_unequal_values_when_rendering_evidence_then_bounds_are_explicit() { + let long = |tail: &str| format!("{}{tail}", "a".repeat(200)); + let test_cases = [ + EvidenceTestCase { + description: "late text difference keeps nearby context", + left: long("left-tail"), + right: long("right-tail"), + options: DiffExampleOptions { + max_value_length: Some(30), + suppress_values: false, + }, + expected_left: "…aaaaaaaaaaaaaaaaaaaaaleft-tail [truncated; 209 chars]".to_owned(), + expected_right: "…aaaaaaaaaaaaaaaaaaaaright-tail [truncated; 210 chars]".to_owned(), + expected_first_difference: Some(200), + }, + EvidenceTestCase { + description: "suppression retains lengths without values", + left: "left-private-value".to_owned(), + right: "right-private-value".to_owned(), + options: DiffExampleOptions { + max_value_length: Some(30), + suppress_values: true, + }, + expected_left: "".to_owned(), + expected_right: "".to_owned(), + expected_first_difference: Some(0), + }, + EvidenceTestCase { + description: "explicit unlimited rendering preserves complete values", + left: long("left-tail"), + right: long("right-tail"), + options: DiffExampleOptions { + max_value_length: None, + suppress_values: false, + }, + expected_left: long("left-tail"), + expected_right: long("right-tail"), + expected_first_difference: Some(200), + }, + ]; + + for test_case in test_cases { + let (left, right) = + render_example_pair(&test_case.left, &test_case.right, test_case.options); + + assert_eq!( + ( + display_example_value(&left), + display_example_value(&right), + left.first_difference, + ), + ( + test_case.expected_left, + test_case.expected_right, + test_case.expected_first_difference, + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_keys_when_rendering_then_long_keys_are_shortened_with_their_digest() { + let test_cases = [ + KeyTestCase { + description: "ordinary key stays complete", + key: "k".repeat(256), + expected_prefix: "k".repeat(256), + expected_suffix: String::new(), + }, + KeyTestCase { + description: "long key names its length and digest", + key: "k".repeat(300), + expected_prefix: format!("{}… [truncated; 300 chars; sha256:", "k".repeat(256)), + expected_suffix: "]".to_owned(), + }, + ]; + + for test_case in test_cases { + let rendered: String = render_key_value(&test_case.key); + + assert_eq!( + ( + rendered.starts_with(&test_case.expected_prefix), + rendered.ends_with(&test_case.expected_suffix), + ), + (true, true), + "{}: {rendered}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_resolving_outcome_then_exit_code_matches() { + let test_cases = [ + OutcomeTestCase { + description: "identical rows pass", + models: vec![model(Some(changed_rows(0)), &[])], + query_row_comparison: false, + expected_outcome: ("pass", 0), + }, + OutcomeTestCase { + description: "unequal rows are findings", + models: vec![model(Some(changed_rows(1)), &[])], + query_row_comparison: false, + expected_outcome: ("findings", 1), + }, + OutcomeTestCase { + description: "query schema change without row comparison is incomplete", + models: vec![model(None, &["extra"])], + query_row_comparison: true, + expected_outcome: ("incomplete", 2), + }, + OutcomeTestCase { + description: "model schema change is a finding", + models: vec![model(None, &["extra"])], + query_row_comparison: false, + expected_outcome: ("findings", 1), + }, + ]; + + for test_case in test_cases { + let (outcome, code) = diff_exit_code(&test_case.models, test_case.query_row_comparison); + + assert_eq!( + (outcome.as_str(), code), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_changed_rows_when_rendering_summary_then_layout_matches_rich() { + let test_cases = [SummaryTestCase { + description: "one unequal row", + model: model(Some(changed_rows(1)), &[]), + expected_lines: &[ + " Model orders ", + "┏━┯━┯━┯━┯━┓╌╌╌┏━┯━┯━┯━┯━┓╭──┬──────────┬─────────╮╌╮ ", + "┃ │ │ │ │ ┃ ≠ ┃ │ │ │ │ ┃│ 2│ equal │ (66.67%)│ │ ", + "│ amount │ mismatches=1 │ match=66.67% │", + ], + }]; + + for test_case in test_cases { + let text: String = render_diff_text( + &[test_case.model], + &document(), + &DiffTextOptions { + mode_label: "full".to_owned(), + max_column_examples: 3, + max_row_only_examples: 3, + ..DiffTextOptions::default() + }, + ); + + let rendered: Vec<&str> = text.lines().collect(); + let found: Vec<&str> = test_case + .expected_lines + .iter() + .copied() + .filter(|line| rendered.contains(line)) + .collect(); + + assert_eq!( + found, + test_case.expected_lines.to_vec(), + "{}: {text}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_rendering_json_then_documents_are_sorted_and_safe() { + let test_cases = [ + JsonTestCase { + description: "non-finite cursor bound becomes a tagged object", + document: render_diff_json(&[model(Some(cursor_rows()), &[])], &document()), + expected_fragment: "\"start\": {\n \"kind\": \"non_finite_number\",\n \"value\": \"NaN\"\n }", + }, + JsonTestCase { + description: "error document keeps the incomplete outcome", + document: render_diff_error_json(&DiffErrorFacts { + status: "incomplete".to_owned(), + code: "C254".to_owned(), + message: "key is not unique".to_owned(), + help: None, + size_guard: None, + }), + expected_fragment: "{\n \"error\": {\n \"code\": \"C254\",\n \"message\": \"key is not unique\"\n },\n \"models\": [],\n \"outcome\": \"incomplete\",\n \"query_comparisons\": [],\n \"schema_version\": 1,\n \"status\": \"incomplete\"\n}", + }, + ]; + + for test_case in test_cases { + let text: String = test_case.document.unwrap_or_default(); + + assert!( + text.contains(test_case.expected_fragment), + "{}: {text}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/diff_report/tests/test_types.rs b/crates/sqlbuild-cli/src/diff_report/tests/test_types.rs new file mode 100644 index 0000000000..c28b5ba3d5 --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/tests/test_types.rs @@ -0,0 +1,76 @@ +use sqlbuild_core::json::errors::JsonEmitError; + +use crate::diff_report::models::{DiffExampleOptions, DiffModelFacts, DiffRequestFacts}; + +/// Two unequal values, how they render, and what the rendering reports. +pub(crate) struct EvidenceTestCase { + pub(crate) description: &'static str, + pub(crate) left: String, + pub(crate) right: String, + pub(crate) options: DiffExampleOptions, + pub(crate) expected_left: String, + pub(crate) expected_right: String, + pub(crate) expected_first_difference: Option, +} + +/// Diff results and the outcome and exit code they produce. +pub(crate) struct OutcomeTestCase { + pub(crate) description: &'static str, + pub(crate) models: Vec, + pub(crate) query_row_comparison: bool, + pub(crate) expected_outcome: (&'static str, i32), +} + +/// One model and fragments its terminal summary must contain. +pub(crate) struct SummaryTestCase { + pub(crate) description: &'static str, + pub(crate) model: DiffModelFacts, + pub(crate) expected_lines: &'static [&'static str], +} + +/// A key value and how its rendering starts and ends. +pub(crate) struct KeyTestCase { + pub(crate) description: &'static str, + pub(crate) key: String, + pub(crate) expected_prefix: String, + pub(crate) expected_suffix: String, +} + +/// A rendered JSON document and a fragment it must contain. +pub(crate) struct JsonTestCase { + pub(crate) description: &'static str, + pub(crate) document: Result, + pub(crate) expected_fragment: &'static str, +} + +/// A positional target range and its parsed targets or error code. +pub(crate) struct NameRangeTestCase { + pub(crate) description: &'static str, + pub(crate) range: &'static str, + pub(crate) expected_targets: Result<(String, Option), String>, +} + +/// Request flags and the code of the first rule they violate. +pub(crate) struct RequestTestCase { + pub(crate) description: &'static str, + pub(crate) request: DiffRequestFacts, + pub(crate) expected_code: Option<&'static str>, +} + +/// A blocked model's request scope and the guard's replacement commands. +pub(crate) struct SizeGuardTestCase { + pub(crate) description: &'static str, + pub(crate) request: DiffRequestFacts, + pub(crate) has_cursor: bool, + pub(crate) expected_full: &'static str, + pub(crate) expected_bounded: Option<&'static str>, + pub(crate) expected_fragments: &'static [&'static str], +} + +/// Query columns, a column limit and the preflight error code. +pub(crate) struct ColumnsTestCase { + pub(crate) description: &'static str, + pub(crate) columns: &'static [&'static str], + pub(crate) max_columns: Option, + pub(crate) expected_outcome: Result<(), (&'static str, &'static str)>, +} diff --git a/crates/sqlbuild-cli/src/diff_report/types.rs b/crates/sqlbuild-cli/src/diff_report/types.rs new file mode 100644 index 0000000000..9cdef6303f --- /dev/null +++ b/crates/sqlbuild-cli/src/diff_report/types.rs @@ -0,0 +1,19 @@ +//! Tuple shapes of diff facts. + +use crate::diff_report::models::DiffValue; + +/// One named value, such as a key column of a sampled row. +pub type DiffNamedValue = (String, DiffValue); + +/// A schema column whose type differs: name, `from` type and `to` type. +pub type DiffTypeChange = (String, String, String); + +/// The phase of a raw-query diff whose user error is being classified. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum QueryDiffPhase { + Preparation, + Execution, +} + +/// Seconds spent in each named phase of a raw-query diff. +pub type PhaseSeconds = Vec<(String, f64)>; diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 450b6d7bf1..b46c6facef 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -240,9 +240,9 @@ const TEST_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const TEST_ROUTE: Route = routed( "run_test", - ("sqlbuild.cli.commands.main.execution._test", "run_test"), + ("sqlbuild._native", "run_test_command"), &TEST_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const CHECK_FIELDS: [Field; 9] = [ @@ -262,9 +262,9 @@ const CHECK_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const CHECK_ROUTE: Route = routed( "run_check", - ("sqlbuild.cli.commands.main.execution._check", "run_check"), + ("sqlbuild._native", "run_check_command"), &CHECK_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const AUDIT_FIELDS: [Field; 11] = [ @@ -310,9 +310,9 @@ const LOAD_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const LOAD_ROUTE: Route = routed( "run_load", - ("sqlbuild.cli.commands.main.execution._load", "run_load"), + ("sqlbuild._native", "run_load_command"), &LOAD_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const SEED_FIELDS: [Field; 9] = [ @@ -332,9 +332,9 @@ const SEED_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const SEED_ROUTE: Route = routed( "run_seed", - ("sqlbuild.cli.commands.main.execution._seed", "run_seed"), + ("sqlbuild._native", "run_seed_command"), &SEED_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const LINEAGE_FIELDS: [Field; 11] = [ @@ -387,9 +387,9 @@ const CLONE_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const CLONE_ROUTE: Route = routed( "run_clone", - ("sqlbuild.cli.commands.main.inspection._clone", "run_clone"), + ("sqlbuild._native", "run_clone_command"), &CLONE_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const DIFF_FIELDS: [Field; 35] = [ @@ -438,9 +438,9 @@ const DIFF_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const DIFF_ROUTE: Route = routed( "run_diff", - ("sqlbuild.cli.commands.main.inspection._diff", "run_diff"), + ("sqlbuild._native", "run_diff_command"), &DIFF_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const QUERY_ARGUMENTS: [Field; 6] = [ diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 01a09fdfc7..4bbb72fffb 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -4,6 +4,7 @@ pub mod argv; pub mod compute_logs; +pub mod diff_report; pub mod dispatch; pub mod launcher; pub mod replay; diff --git a/crates/sqlbuild-config/src/dbt_manifest/_helpers/fields.rs b/crates/sqlbuild-config/src/dbt_manifest/_helpers/fields.rs new file mode 100644 index 0000000000..94656221c0 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/_helpers/fields.rs @@ -0,0 +1,44 @@ +//! Node field rules shared by models, sources and seeds. + +use crate::dbt_manifest::constants::DBT_REF_FUNCTION; +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::models::ManifestNode; + +/// A non-empty string field. +pub(crate) fn optional(node: &ManifestNode, field: &str) -> Option { + node.text + .iter() + .find(|(key, value)| key == field && !value.is_empty()) + .map(|(_, value)| value.clone()) +} + +/// A non-empty string field, or the invalid-manifest error naming it. +pub(crate) fn required(node: &ManifestNode, field: &str) -> Result { + optional(node, field) + .ok_or_else(|| ManifestError::invalid(&format!(": model {field} is required"))) +} + +/// The given relation name, or `database.schema.name` without empty parts. +pub(crate) fn relation_name( + node: &ManifestNode, + (database, schema): (&Option, &Option), + name: &str, +) -> String { + optional(node, "relation_name").unwrap_or_else(|| { + [database.as_deref(), schema.as_deref(), Some(name)] + .into_iter() + .flatten() + .filter(|part| !part.is_empty()) + .collect::>() + .join(".") + }) +} + +/// `SqlReferenceKind.DBT_REF.example_call(*args, quote='"')`. +pub(crate) fn quoted_dbt_ref(args: &[&str]) -> String { + let quoted: Vec = args + .iter() + .map(|arg| format!("\"{}\"", arg.replace('"', "\"\""))) + .collect(); + format!("{DBT_REF_FUNCTION}({})", quoted.join(", ")) +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/_helpers/mod.rs b/crates/sqlbuild-config/src/dbt_manifest/_helpers/mod.rs new file mode 100644 index 0000000000..f875d0dd21 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod fields; diff --git a/crates/sqlbuild-config/src/dbt_manifest/constants.rs b/crates/sqlbuild-config/src/dbt_manifest/constants.rs new file mode 100644 index 0000000000..4ec61bd069 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/constants.rs @@ -0,0 +1,27 @@ +//! dbt manifest resource types and the help SQLBuild shows for manifest problems. + +pub const MODEL_RESOURCE_TYPE: &str = "model"; +pub const SNAPSHOT_RESOURCE_TYPE: &str = "snapshot"; +pub const SEED_RESOURCE_TYPE: &str = "seed"; +pub const SOURCE_RESOURCE_TYPE: &str = "source"; +pub const TEST_RESOURCE_TYPE: &str = "test"; +pub const UNIT_TEST_RESOURCE_TYPE: &str = "unit_test"; +/// Node fields read as text: identity, namespace, relation and SQL. +pub const NODE_TEXT_FIELDS: [&str; 13] = [ + "package_name", + "name", + "source_name", + "database", + "schema", + "alias", + "identifier", + "relation_name", + "raw_code", + "compiled_code", + "raw_sql", + "compiled_sql", + "resource_type", +]; +pub(crate) const MANIFEST_HELP: &str = + "Run dbt compile or configure dbt target_path so SQLBuild can read manifest.json."; +pub(crate) const DBT_REF_FUNCTION: &str = "__dbt_ref"; diff --git a/crates/sqlbuild-config/src/dbt_manifest/errors.rs b/crates/sqlbuild-config/src/dbt_manifest/errors.rs new file mode 100644 index 0000000000..2aff1c52c3 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/errors.rs @@ -0,0 +1,97 @@ +//! A dbt manifest SQLBuild cannot read or resolve against. + +use crate::dbt_manifest::_helpers::fields::quoted_dbt_ref; +use crate::dbt_manifest::constants::{DBT_REF_FUNCTION, MANIFEST_HELP}; + +/// `CompileInputError(message, code=code, help=help)`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestError { + pub code: &'static str, + pub message: String, + pub help: Option, +} + +impl ManifestError { + /// No model `name` (in `package_name`, when given) exists in the manifest. + #[must_use] + pub fn unknown_model(name: &str, package_name: Option<&str>) -> Self { + let (target, what) = match package_name { + Some(package) => (format!("{package}.{name}"), "package/name"), + None => (name.to_owned(), "name"), + }; + Self { + code: "C212", + message: format!("dbt model '{target}' was not found in manifest"), + help: Some(format!( + "Check the {DBT_REF_FUNCTION}(...) {what} or run dbt compile to refresh manifest.json." + )), + } + } + + /// `name` matches models in several packages, listed sorted. + #[must_use] + pub fn ambiguous_model(name: &str, mut packages: Vec) -> Self { + packages.sort(); + Self { + code: "C213", + message: format!( + "dbt model '{name}' is ambiguous across packages: {}", + packages.join(", ") + ), + help: Some(format!( + "Use {} to choose one dbt model.", + quoted_dbt_ref(&["package_name", name]) + )), + } + } + + /// A model uses `__dbt_ref` but no manifest was found. + #[must_use] + pub fn missing_manifest(model_relative_path: &str, ref_name: &str) -> Self { + Self { + code: "C214", + message: format!( + "Model file {model_relative_path} uses {} but no dbt manifest was found", + quoted_dbt_ref(&[ref_name]) + ), + help: Some(MANIFEST_HELP.to_owned()), + } + } + + /// dbt and SQLBuild models share these names, sorted. + #[must_use] + pub fn shared_model_names(mut names: Vec) -> Option { + if names.is_empty() { + return None; + } + names.sort(); + Some(Self { + code: "C215", + message: format!("dbt and SQLBuild models share names: {}", names.join(", ")), + help: Some( + "Rename either the dbt model or SQLBuild model; owner-qualified names are not supported." + .to_owned(), + ), + }) + } + + /// The manifest file does not exist. + #[must_use] + pub fn missing_manifest_file(path: &str) -> Self { + Self { + code: "C210", + message: format!("dbt manifest file does not exist: {path}"), + help: Some(MANIFEST_HELP.to_owned()), + } + } + + /// The manifest is not the JSON shape SQLBuild reads. + #[must_use] + pub fn invalid(detail: &str) -> Self { + Self { + code: "C211", + message: format!("Invalid dbt manifest{detail}"), + help: None, + } + } +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/main/mod.rs b/crates/sqlbuild-config/src/dbt_manifest/main/mod.rs new file mode 100644 index 0000000000..e674181aa6 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/main/mod.rs @@ -0,0 +1,3 @@ +pub mod parse_manifest_model; +pub mod parse_manifest_seed; +pub mod parse_manifest_source; diff --git a/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_model.rs b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_model.rs new file mode 100644 index 0000000000..89ddd401a8 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_model.rs @@ -0,0 +1,56 @@ +//! A dbt model or snapshot node. + +use crate::dbt_manifest::_helpers::fields::{optional, relation_name, required}; +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::models::{ManifestNode, NodeChecksum, ParsedModel}; + +/// Every part when each is a non-empty string. +fn fqn_parts(parts: Vec>) -> Option> { + let mut names: Vec = Vec::new(); + for part in parts { + names.push(part.filter(|part| !part.is_empty())?); + } + Some(names) +} + +/// The model's identity, namespace, relation, checksum, fqn, query SQL and dependencies. +pub fn parse_manifest_model(node: &ManifestNode) -> Result { + let package_name: String = required(node, "package_name")?; + let name: String = required(node, "name")?; + let database: Option = optional(node, "database"); + let schema: Option = optional(node, "schema"); + let alias: Option = optional(node, "alias"); + let relation: String = relation_name( + node, + (&database, &schema), + alias.as_deref().unwrap_or(&name), + ); + let fqn: Vec = node.fqn.clone().and_then(fqn_parts).unwrap_or_default(); + let query_sql: String = ["raw_code", "compiled_code", "raw_sql", "compiled_sql"] + .into_iter() + .find_map(|field| optional(node, field)) + .unwrap_or_default(); + let node_checksum: Option = match &node.checksum { + NodeChecksum::Text(text) | NodeChecksum::Nested(Some(text)) if !text.is_empty() => { + Some(text.clone()) + } + _ => None, + }; + Ok(ParsedModel { + package_name, + name, + database, + schema, + alias, + node_checksum, + relation_name: relation, + fqn, + query_sql, + depends_on: node + .depends_on + .iter() + .filter(|unique_id| !unique_id.is_empty()) + .cloned() + .collect(), + }) +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_seed.rs b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_seed.rs new file mode 100644 index 0000000000..873bec4935 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_seed.rs @@ -0,0 +1,27 @@ +//! A dbt seed node. + +use crate::dbt_manifest::_helpers::fields::{optional, relation_name, required}; +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::models::{ManifestNode, ParsedSeed}; + +/// The seed's identity, namespace and relation. +pub fn parse_manifest_seed(node: &ManifestNode) -> Result { + let package_name: String = required(node, "package_name")?; + let name: String = required(node, "name")?; + let database: Option = optional(node, "database"); + let schema: Option = optional(node, "schema"); + let alias: Option = optional(node, "alias"); + let relation: String = relation_name( + node, + (&database, &schema), + alias.as_deref().unwrap_or(&name), + ); + Ok(ParsedSeed { + package_name, + name, + database, + schema, + alias, + relation_name: relation, + }) +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_source.rs b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_source.rs new file mode 100644 index 0000000000..e6a567aa1f --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/main/parse_manifest_source.rs @@ -0,0 +1,29 @@ +//! A dbt source node. + +use crate::dbt_manifest::_helpers::fields::{optional, relation_name, required}; +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::models::{ManifestNode, ParsedSource}; + +/// The source's identity, namespace and relation. +pub fn parse_manifest_source(node: &ManifestNode) -> Result { + let package_name: String = required(node, "package_name")?; + let source_name: String = required(node, "source_name")?; + let name: String = required(node, "name")?; + let database: Option = optional(node, "database"); + let schema: Option = optional(node, "schema"); + let identifier: Option = optional(node, "identifier"); + let relation: String = relation_name( + node, + (&database, &schema), + identifier.as_deref().unwrap_or(&name), + ); + Ok(ParsedSource { + package_name, + source_name, + name, + database, + schema, + identifier, + relation_name: relation, + }) +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/mod.rs b/crates/sqlbuild-config/src/dbt_manifest/mod.rs new file mode 100644 index 0000000000..899237ebcb --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/dbt_manifest/models.rs b/crates/sqlbuild-config/src/dbt_manifest/models.rs new file mode 100644 index 0000000000..944ee3f3dd --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/models.rs @@ -0,0 +1,63 @@ +//! dbt manifest node fields and the parsed models, sources and seeds. + +/// A node `checksum` field. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum NodeChecksum { + /// A string value. + Text(String), + /// An object with this string `checksum` member, when it has one. + Nested(Option), + /// Missing or any other value. + #[default] + Other, +} + +/// The fields of one manifest node that SQLBuild reads. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ManifestNode { + /// `NODE_TEXT_FIELDS` whose value is a string, with that value. + pub text: Vec<(String, String)>, + pub checksum: NodeChecksum, + /// `fqn` when it is a list: each item when it is a string. + pub fqn: Option>>, + /// The string items of `depends_on.nodes`. + pub depends_on: Vec, +} + +/// A dbt model or snapshot node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParsedModel { + pub package_name: String, + pub name: String, + pub database: Option, + pub schema: Option, + pub alias: Option, + pub node_checksum: Option, + pub relation_name: String, + pub fqn: Vec, + pub query_sql: String, + pub depends_on: Vec, +} + +/// A dbt source node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParsedSource { + pub package_name: String, + pub source_name: String, + pub name: String, + pub database: Option, + pub schema: Option, + pub identifier: Option, + pub relation_name: String, +} + +/// A dbt seed node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParsedSeed { + pub package_name: String, + pub name: String, + pub database: Option, + pub schema: Option, + pub alias: Option, + pub relation_name: String, +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/tests.rs b/crates/sqlbuild-config/src/dbt_manifest/tests.rs new file mode 100644 index 0000000000..74248b83bd --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dbt_manifest.rs"] +mod dbt_manifest; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/dbt_manifest/tests/helpers.rs b/crates/sqlbuild-config/src/dbt_manifest/tests/helpers.rs new file mode 100644 index 0000000000..764e02e8a5 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/tests/helpers.rs @@ -0,0 +1,16 @@ +use crate::dbt_manifest::models::{ManifestNode, NodeChecksum}; + +pub(super) fn node(text: &[(&str, &str)], checksum: NodeChecksum) -> ManifestNode { + ManifestNode { + text: text + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(), + checksum, + fqn: Some(vec![ + Some("analytics".to_owned()), + Some("orders".to_owned()), + ]), + depends_on: vec!["model.a.b".to_owned(), String::new()], + } +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/tests/test_dbt_manifest.rs b/crates/sqlbuild-config/src/dbt_manifest/tests/test_dbt_manifest.rs new file mode 100644 index 0000000000..1557b19457 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/tests/test_dbt_manifest.rs @@ -0,0 +1,59 @@ +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::main::parse_manifest_model::parse_manifest_model; +use crate::dbt_manifest::models::{NodeChecksum, ParsedModel}; +use crate::dbt_manifest::tests::helpers::node; +use crate::dbt_manifest::tests::test_types::ModelTestCase; + +#[test] +fn given_manifest_model_nodes_when_parsing_then_matches_python_rules() { + let test_cases = [ + ModelTestCase { + description: "relation from namespace and alias, nested checksum, compiled sql fallback", + node: node( + &[ + ("package_name", "analytics"), + ("name", "orders"), + ("database", ""), + ("schema", "main"), + ("alias", "orders_v2"), + ("raw_code", ""), + ("compiled_code", "select 1"), + ], + NodeChecksum::Nested(Some("abc".to_owned())), + ), + expected_result: Ok(ParsedModel { + package_name: "analytics".to_owned(), + name: "orders".to_owned(), + database: None, + schema: Some("main".to_owned()), + alias: Some("orders_v2".to_owned()), + node_checksum: Some("abc".to_owned()), + relation_name: "main.orders_v2".to_owned(), + fqn: vec!["analytics".to_owned(), "orders".to_owned()], + query_sql: "select 1".to_owned(), + depends_on: vec!["model.a.b".to_owned()], + }), + }, + ModelTestCase { + description: "empty name is rejected", + node: node( + &[("package_name", "analytics"), ("name", "")], + NodeChecksum::Other, + ), + expected_result: Err(ManifestError { + code: "C211", + message: "Invalid dbt manifest: model name is required".to_owned(), + help: None, + }), + }, + ]; + + for test_case in test_cases { + assert_eq!( + parse_manifest_model(&test_case.node), + test_case.expected_result, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/dbt_manifest/tests/test_types.rs b/crates/sqlbuild-config/src/dbt_manifest/tests/test_types.rs new file mode 100644 index 0000000000..712d99f625 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_manifest/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::dbt_manifest::errors::ManifestError; +use crate::dbt_manifest::models::{ManifestNode, ParsedModel}; + +pub(super) struct ModelTestCase { + pub(super) description: &'static str, + pub(super) node: ManifestNode, + pub(super) expected_result: Result, +} diff --git a/crates/sqlbuild-config/src/dbt_profile/_helpers/adapters.rs b/crates/sqlbuild-config/src/dbt_profile/_helpers/adapters.rs new file mode 100644 index 0000000000..33f8cb933e --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/_helpers/adapters.rs @@ -0,0 +1,209 @@ +//! Per-adapter normalization of a rendered dbt profile output. + +use crate::dbt_profile::_helpers::output::Output; +use crate::dbt_profile::constants::{ + BIGQUERY_METHOD_ERROR_SUFFIX, BIGQUERY_SUPPORTED_METHODS, BIGQUERY_UNSUPPORTED_METHODS, + DATABRICKS_OAUTH_KEYS, DATABRICKS_PAT_AUTH_TYPE, DEFAULT_DATABRICKS_CATALOG, + DUCKDB_COPIED_KEYS, DUCKDB_UNSUPPORTED_KEYS, MEMORY_DATABASE, POSTGRES_COPIED_KEYS, + SNOWFLAKE_DROPPED_KEYS, SQLSERVER_SQL_AUTHENTICATION, +}; +use crate::dbt_profile::errors::DbtProfileError; +use crate::dbt_profile::models::{ConnectionKey, ConnectionSource, NormalizedProfile}; + +fn normalized( + adapter: &'static str, + connection: Vec<(ConnectionKey, ConnectionSource)>, +) -> NormalizedProfile { + NormalizedProfile { + adapter, + connection, + target_schema: None, + target_database: None, + warnings: Vec::new(), + } +} + +fn copied(output: &Output<'_>, keys: &[&'static str]) -> Vec<(ConnectionKey, ConnectionSource)> { + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = Vec::new(); + for key in keys { + if let Some(index) = output.index(key) { + connection.push((ConnectionKey::Name(key), ConnectionSource::Entry(index))); + } + } + connection +} + +fn renamed( + output: &Output<'_>, + (from, to): (&str, &'static str), +) -> Option<(ConnectionKey, ConnectionSource)> { + output + .index(from) + .map(|index| (ConnectionKey::Name(to), ConnectionSource::Entry(index))) +} + +pub(crate) fn duckdb(output: &Output<'_>) -> NormalizedProfile { + let database: ConnectionSource = output + .first_value(&["path"]) + .or_else(|| output.first_value(&["database"])) + .map_or( + ConnectionSource::Text(MEMORY_DATABASE), + ConnectionSource::Entry, + ); + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = + vec![(ConnectionKey::Name("database"), database)]; + connection.extend(copied(output, &DUCKDB_COPIED_KEYS)); + let unsupported: Vec = output.sorted_present(&DUCKDB_UNSUPPORTED_KEYS, false); + let mut profile: NormalizedProfile = normalized("duckdb", connection); + if !unsupported.is_empty() { + profile.warnings.push(format!( + "dbt-duckdb profile fields are not applied by SQLBuild yet: {}", + unsupported.join(", ") + )); + } + profile.target_schema = output.text_at(output.index("schema")); + profile.target_database = output.text_at(output.index("database")); + profile +} + +pub(crate) fn postgres(output: &Output<'_>) -> NormalizedProfile { + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = + copied(output, &POSTGRES_COPIED_KEYS); + if let Some(index) = output.first_value(&["dbname", "database"]) { + connection.push(( + ConnectionKey::Name("dbname"), + ConnectionSource::Entry(index), + )); + } + if let Some(index) = output.first_value(&["pass", "password"]) { + connection.push(( + ConnectionKey::Name("password"), + ConnectionSource::Entry(index), + )); + } + let mut profile: NormalizedProfile = normalized("postgres", connection); + profile.target_schema = output.non_empty_text(&["schema"]); + profile +} + +pub(crate) fn snowflake(output: &Output<'_>) -> NormalizedProfile { + let connection: Vec<(ConnectionKey, ConnectionSource)> = output + .0 + .iter() + .enumerate() + .filter(|(_, entry)| { + !entry + .key + .as_deref() + .is_some_and(|key| SNOWFLAKE_DROPPED_KEYS.contains(&key)) + }) + .map(|(index, _)| (ConnectionKey::Entry(index), ConnectionSource::Entry(index))) + .collect(); + let mut profile: NormalizedProfile = normalized("snowflake", connection); + profile.target_schema = output.non_empty_text(&["schema"]); + profile.target_database = output.non_empty_text(&["database"]); + profile +} + +fn bigquery_method_error(method: &str) -> DbtProfileError { + DbtProfileError(format!( + "dbt BigQuery profile method '{method}' {BIGQUERY_METHOD_ERROR_SUFFIX}" + )) +} + +pub(crate) fn bigquery(output: &Output<'_>) -> Result { + let method: Option = output.non_empty_text(&["method"]); + if let Some(method) = &method + && BIGQUERY_UNSUPPORTED_METHODS.contains(&method.as_str()) + { + return Err(bigquery_method_error(method)); + } + let project: Option = output.first_value(&["project", "database"]); + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = Vec::new(); + if let Some(index) = project { + connection.push(( + ConnectionKey::Name("project"), + ConnectionSource::Entry(index), + )); + } + connection.extend(renamed(output, ("location", "location"))); + connection.extend(renamed(output, ("keyfile", "credentials_path"))); + if let Some(method) = &method + && !BIGQUERY_SUPPORTED_METHODS.contains(&method.as_str()) + { + return Err(bigquery_method_error(method)); + } + let mut profile: NormalizedProfile = normalized("bigquery", connection); + profile.target_schema = output.non_empty_text(&["dataset", "schema"]); + profile.target_database = output.text_at(project).filter(|text| !text.is_empty()); + Ok(profile) +} + +pub(crate) fn databricks(output: &Output<'_>) -> Result { + let unsupported: Vec = output.sorted_present(&DATABRICKS_OAUTH_KEYS, true); + if let Some(auth_type) = output.non_empty_text(&["auth_type"]) + && auth_type != DATABRICKS_PAT_AUTH_TYPE + { + return Err(DbtProfileError(format!( + "dbt Databricks profile auth_type '{auth_type}' is not supported by SQLBuild dbt-profile normalization yet. Supported auth: personal access token" + ))); + } + if !unsupported.is_empty() { + return Err(DbtProfileError(format!( + "dbt Databricks OAuth/Azure profile fields are not supported by SQLBuild dbt-profile normalization yet: {}", + unsupported.join(", ") + ))); + } + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = Vec::new(); + connection.extend(renamed(output, ("host", "server_hostname"))); + connection.extend(renamed(output, ("http_path", "http_path"))); + connection.extend(renamed(output, ("token", "token"))); + let catalog: Option = output.first_value(&["catalog", "database"]); + connection.push(( + ConnectionKey::Name("catalog"), + catalog.map_or( + ConnectionSource::Text(DEFAULT_DATABRICKS_CATALOG), + ConnectionSource::Entry, + ), + )); + connection.extend(renamed(output, ("schema", "schema"))); + let mut profile: NormalizedProfile = normalized("databricks", connection); + profile.target_schema = output.non_empty_text(&["schema"]); + profile.target_database = match catalog { + Some(index) => output.text_at(Some(index)).filter(|text| !text.is_empty()), + None => Some(DEFAULT_DATABRICKS_CATALOG.to_owned()), + }; + Ok(profile) +} + +pub(crate) fn sqlserver(output: &Output<'_>) -> Result { + if output.is_true("windows_login") || output.is_true("trusted_connection") { + return Err(DbtProfileError( + "dbt SQL Server Windows authentication is not supported by SQLBuild dbt-profile normalization yet" + .to_owned(), + )); + } + if let Some(authentication) = output.non_empty_text(&["authentication", "auth"]) + && authentication.to_lowercase() != SQLSERVER_SQL_AUTHENTICATION + { + return Err(DbtProfileError(format!( + "dbt SQL Server authentication mode '{authentication}' is not supported by SQLBuild dbt-profile normalization yet. Supported authentication: sql" + ))); + } + let values: [(&'static str, Option); 5] = [ + ("host", output.first_value(&["server", "host"])), + ("port", output.first_value(&["port"])), + ("user", output.first_value(&["UID", "user", "username"])), + ("password", output.first_value(&["PWD", "password", "pass"])), + ("database", output.first_value(&["database"])), + ]; + let mut connection: Vec<(ConnectionKey, ConnectionSource)> = Vec::new(); + for (key, index) in values { + if let Some(index) = index { + connection.push((ConnectionKey::Name(key), ConnectionSource::Entry(index))); + } + } + let mut profile: NormalizedProfile = normalized("sqlserver", connection); + profile.target_schema = output.non_empty_text(&["schema"]); + Ok(profile) +} diff --git a/crates/sqlbuild-config/src/dbt_profile/_helpers/mod.rs b/crates/sqlbuild-config/src/dbt_profile/_helpers/mod.rs new file mode 100644 index 0000000000..ded0f58ac0 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod adapters; +pub(crate) mod output; diff --git a/crates/sqlbuild-config/src/dbt_profile/_helpers/output.rs b/crates/sqlbuild-config/src/dbt_profile/_helpers/output.rs new file mode 100644 index 0000000000..4aadeb29ec --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/_helpers/output.rs @@ -0,0 +1,59 @@ +//! Lookups over a profile output with Python `dict` semantics. + +use crate::dbt_profile::models::{ProfileEntry, ProfileValue}; + +/// A profile output, read as Python reads the rendered `dict`. +pub(crate) struct Output<'a>(pub(crate) &'a [ProfileEntry]); + +impl Output<'_> { + /// The index of `key`, when present. + pub(crate) fn index(&self, key: &str) -> Option { + self.0 + .iter() + .position(|entry| entry.key.as_deref() == Some(key)) + } + + /// `output.get(keys[0], output.get(keys[1], ...))`: the first present key's index. + pub(crate) fn first_present(&self, keys: &[&str]) -> Option { + keys.iter().find_map(|key| self.index(key)) + } + + /// The first present key's index, unless its value is `None`. + pub(crate) fn first_value(&self, keys: &[&str]) -> Option { + self.first_present(keys) + .filter(|index| self.0[*index].value != ProfileValue::Null) + } + + /// `_string_or_none(output.get(...))`: a non-empty string value. + pub(crate) fn non_empty_text(&self, keys: &[&str]) -> Option { + self.text_at(self.first_present(keys)) + .filter(|text| !text.is_empty()) + } + + /// The string value at `index`, when it is a string. + pub(crate) fn text_at(&self, index: Option) -> Option { + match &self.0[index?].value { + ProfileValue::Text(text) => Some(text.clone()), + _ => None, + } + } + + /// Whether the value of `key` is exactly `True`. + pub(crate) fn is_true(&self, key: &str) -> bool { + self.index(key) + .is_some_and(|index| self.0[index].value == ProfileValue::Bool(true)) + } + + /// Named keys present in the output that are listed in `keys`, sorted. + pub(crate) fn sorted_present(&self, keys: &[&str], with_value: bool) -> Vec { + let mut present: Vec = self + .0 + .iter() + .filter(|entry| !with_value || entry.value != ProfileValue::Null) + .filter_map(|entry| entry.key.clone()) + .filter(|key| keys.contains(&key.as_str())) + .collect(); + present.sort_unstable(); + present + } +} diff --git a/crates/sqlbuild-config/src/dbt_profile/constants.rs b/crates/sqlbuild-config/src/dbt_profile/constants.rs new file mode 100644 index 0000000000..12b144478c --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/constants.rs @@ -0,0 +1,50 @@ +//! dbt profile output keys SQLBuild reads and the wording of unsupported profiles. + +pub(crate) const MEMORY_DATABASE: &str = ":memory:"; +pub(crate) const DEFAULT_DATABRICKS_CATALOG: &str = "hive_metastore"; +pub(crate) const DATABRICKS_PAT_AUTH_TYPE: &str = "pat"; +pub(crate) const SQLSERVER_SQL_AUTHENTICATION: &str = "sql"; +pub(crate) const BIGQUERY_SUPPORTED_METHODS: [&str; 2] = ["oauth", "service-account"]; +pub(crate) const BIGQUERY_UNSUPPORTED_METHODS: [&str; 3] = [ + "oauth-secrets", + "service-account-json", + "external-oauth-wif", +]; +pub(crate) const DUCKDB_COPIED_KEYS: [&str; 3] = ["extensions", "settings", "attach"]; +pub(crate) const DUCKDB_UNSUPPORTED_KEYS: [&str; 12] = [ + "config_options", + "disable_transactions", + "external_root", + "filesystems", + "is_ducklake", + "keep_open", + "module_paths", + "plugins", + "remote", + "retries", + "secrets", + "use_credential_provider", +]; +pub(crate) const POSTGRES_COPIED_KEYS: [&str; 11] = [ + "host", + "port", + "user", + "connect_timeout", + "role", + "keepalives_idle", + "sslmode", + "sslcert", + "sslkey", + "sslrootcert", + "application_name", +]; +pub(crate) const SNOWFLAKE_DROPPED_KEYS: [&str; 2] = ["type", "threads"]; +pub(crate) const DATABRICKS_OAUTH_KEYS: [&str; 6] = [ + "client_id", + "client_secret", + "azure_client_id", + "azure_client_secret", + "oauth_redirect_url", + "oauth_scopes", +]; +pub(crate) const BIGQUERY_METHOD_ERROR_SUFFIX: &str = "is not supported by SQLBuild dbt-profile normalization yet. Supported methods: oauth, service-account"; diff --git a/crates/sqlbuild-config/src/dbt_profile/errors.rs b/crates/sqlbuild-config/src/dbt_profile/errors.rs new file mode 100644 index 0000000000..fc3c81b94a --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/errors.rs @@ -0,0 +1,5 @@ +//! A dbt profile SQLBuild cannot connect with. + +/// `DbtProfileError(message)`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DbtProfileError(pub String); diff --git a/crates/sqlbuild-config/src/dbt_profile/main/dbt_init_project_toml.rs b/crates/sqlbuild-config/src/dbt_profile/main/dbt_init_project_toml.rs new file mode 100644 index 0000000000..f8ab22a2d1 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/main/dbt_init_project_toml.rs @@ -0,0 +1,50 @@ +//! The `sqlbuild_project.toml` that `sqb dbt init` writes. + +use sqlbuild_core::text::main::python_isalnum::python_isalnum; + +use crate::dbt_profile::models::DbtInitConfig; + +fn escaped(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") +} + +/// A bare TOML key when it is alphanumeric apart from `_` and `-`, else a quoted one. +fn table_key(value: &str) -> String { + let stripped: String = value.replace(['_', '-'], ""); + if !stripped.is_empty() && stripped.chars().all(python_isalnum) { + return value.to_owned(); + } + format!("\"{}\"", escaped(value)) +} + +/// The project config pointing SQLBuild at the dbt project and its profile target. +#[must_use] +pub fn dbt_init_project_toml(config: &DbtInitConfig) -> String { + let connection_name: String = format!("{}_dbt_profile", config.target_name); + let mut target_lines: Vec = vec![ + format!("[connections.{}]", table_key(&connection_name)), + "source = \"dbt_profile\"".to_owned(), + format!("profile = \"{}\"", escaped(&config.profile_name)), + format!("target = \"{}\"", escaped(&config.target_name)), + String::new(), + format!("[targets.{}]", table_key(&config.target_name)), + format!("connection = \"{}\"", escaped(&connection_name)), + ]; + if let Some(database) = &config.target_database { + target_lines.push(format!("database = \"{}\"", escaped(database))); + } + if let Some(schema) = &config.target_schema { + target_lines.push(format!("schema = \"{}\"", escaped(schema))); + } + format!( + "name = \"{}\"\nadapter = \"{}\"\ndefault_target = \"{}\"\n\n[dbt]\nproject_dir = \"{}\"\nprofiles_dir = \"{}\"\ntarget_path = \"{}\"\ntarget = \"{}\"\n\n{}\n", + escaped(&config.project_name), + escaped(&config.adapter), + escaped(&config.target_name), + escaped(&config.dbt_project_dir), + escaped(&config.profiles_dir), + escaped(&config.target_path), + escaped(&config.target_name), + target_lines.join("\n") + ) +} diff --git a/crates/sqlbuild-config/src/dbt_profile/main/mod.rs b/crates/sqlbuild-config/src/dbt_profile/main/mod.rs new file mode 100644 index 0000000000..d483631463 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/main/mod.rs @@ -0,0 +1,2 @@ +pub mod dbt_init_project_toml; +pub mod normalize_dbt_profile; diff --git a/crates/sqlbuild-config/src/dbt_profile/main/normalize_dbt_profile.rs b/crates/sqlbuild-config/src/dbt_profile/main/normalize_dbt_profile.rs new file mode 100644 index 0000000000..5772c1e9fe --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/main/normalize_dbt_profile.rs @@ -0,0 +1,32 @@ +//! Normalize a rendered dbt profile output into a SQLBuild connection. + +use crate::dbt_profile::_helpers::adapters::{ + bigquery, databricks, duckdb, postgres, snowflake, sqlserver, +}; +use crate::dbt_profile::_helpers::output::Output; +use crate::dbt_profile::errors::DbtProfileError; +use crate::dbt_profile::models::{NormalizedProfile, ProfileEntry}; + +/// The SQLBuild connection plan for a rendered profile output of type `adapter_type`. +pub fn normalize_dbt_profile( + entries: &[ProfileEntry], +) -> Result { + let output: Output<'_> = Output(entries); + let adapter_type: String = output.text_at(output.index("type")).unwrap_or_default(); + match adapter_type.as_str() { + "duckdb" => Ok(duckdb(&output)), + "postgres" => Ok(postgres(&output)), + "snowflake" => Ok(snowflake(&output)), + "bigquery" => bigquery(&output), + "databricks" => databricks(&output), + "sqlserver" => sqlserver(&output), + "motherduck" => Err(DbtProfileError( + "dbt profile type 'motherduck' is not supported directly. Use a dbt-duckdb profile with a MotherDuck path after SQLBuild MotherDuck profile support lands." + .to_owned(), + )), + _ => Err(DbtProfileError(format!( + "dbt profile type '{}' is not supported by SQLBuild dbt-profile connection resolution yet. Currently supported: bigquery, databricks, duckdb, postgres, snowflake, sqlserver", + if adapter_type.is_empty() { "" } else { adapter_type.as_str() } + ))), + } +} diff --git a/crates/sqlbuild-config/src/dbt_profile/mod.rs b/crates/sqlbuild-config/src/dbt_profile/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/dbt_profile/models.rs b/crates/sqlbuild-config/src/dbt_profile/models.rs new file mode 100644 index 0000000000..9f77770ac7 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/models.rs @@ -0,0 +1,61 @@ +//! A rendered dbt profile output and the SQLBuild connection built from it. + +/// One profile output value, as normalization tells values apart. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProfileValue { + /// `None`. + Null, + /// `True` or `False`. + Bool(bool), + Text(String), + /// Any other value; copied through unchanged. + Other, +} + +/// One `key: value` entry of a profile output, in output order; non-string keys have no name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProfileEntry { + pub key: Option, + pub value: ProfileValue, +} + +/// Where one connection value comes from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConnectionSource { + /// The value of the output entry at this index. + Entry(usize), + /// A literal default. + Text(&'static str), +} + +/// The key one connection value is written under. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConnectionKey { + Name(&'static str), + /// The key of the output entry at this index, unchanged. + Entry(usize), +} + +/// A normalized SQLBuild connection: its adapter, ordered values, target namespace and warnings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NormalizedProfile { + pub adapter: &'static str, + pub connection: Vec<(ConnectionKey, ConnectionSource)>, + pub target_schema: Option, + pub target_database: Option, + pub warnings: Vec, +} + +/// The values `sqb dbt init` writes into `sqlbuild_project.toml`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DbtInitConfig { + pub project_name: String, + pub adapter: String, + pub target_name: String, + pub profile_name: String, + pub dbt_project_dir: String, + pub profiles_dir: String, + pub target_path: String, + pub target_schema: Option, + pub target_database: Option, +} diff --git a/crates/sqlbuild-config/src/dbt_profile/tests.rs b/crates/sqlbuild-config/src/dbt_profile/tests.rs new file mode 100644 index 0000000000..003e0bb7d0 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dbt_profile.rs"] +mod dbt_profile; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/dbt_profile/tests/helpers.rs b/crates/sqlbuild-config/src/dbt_profile/tests/helpers.rs new file mode 100644 index 0000000000..ff983129d0 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/tests/helpers.rs @@ -0,0 +1,26 @@ +use crate::dbt_profile::models::{DbtInitConfig, ProfileEntry, ProfileValue}; + +pub(super) fn entry(key: &str, value: ProfileValue) -> ProfileEntry { + ProfileEntry { + key: Some(key.to_owned()), + value, + } +} + +pub(super) fn text(key: &str, value: &str) -> ProfileEntry { + entry(key, ProfileValue::Text(value.to_owned())) +} + +pub(super) fn init_config(target_name: &str, target_schema: Option<&str>) -> DbtInitConfig { + DbtInitConfig { + project_name: "analytics".to_owned(), + adapter: "duckdb".to_owned(), + target_name: target_name.to_owned(), + profile_name: "ana\"lytics".to_owned(), + dbt_project_dir: "../dbt_project".to_owned(), + profiles_dir: "C:\\profiles".to_owned(), + target_path: "../dbt_project/target".to_owned(), + target_schema: target_schema.map(str::to_owned), + target_database: None, + } +} diff --git a/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_profile.rs b/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_profile.rs new file mode 100644 index 0000000000..a16119e002 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_profile.rs @@ -0,0 +1,119 @@ +use crate::dbt_profile::errors::DbtProfileError; +use crate::dbt_profile::main::dbt_init_project_toml::dbt_init_project_toml; +use crate::dbt_profile::main::normalize_dbt_profile::normalize_dbt_profile; +use crate::dbt_profile::models::{ + ConnectionKey, ConnectionSource, NormalizedProfile, ProfileValue, +}; +use crate::dbt_profile::tests::helpers::{entry, init_config, text}; +use crate::dbt_profile::tests::test_types::{InitTomlTestCase, NormalizeTestCase}; + +#[test] +fn given_rendered_profile_outputs_when_normalizing_then_matches_python_rules() { + let test_cases = [ + NormalizeTestCase { + description: "duckdb falls back from a null path to memory and warns on plugins", + entries: vec![ + text("type", "duckdb"), + entry("path", ProfileValue::Null), + entry("plugins", ProfileValue::Other), + entry("extensions", ProfileValue::Other), + text("schema", ""), + ], + expected_result: Ok(NormalizedProfile { + adapter: "duckdb", + connection: vec![ + (ConnectionKey::Name("database"), ConnectionSource::Text(":memory:")), + (ConnectionKey::Name("extensions"), ConnectionSource::Entry(3)), + ], + target_schema: Some(String::new()), + target_database: None, + warnings: vec![ + "dbt-duckdb profile fields are not applied by SQLBuild yet: plugins".to_owned(), + ], + }), + }, + NormalizeTestCase { + description: "postgres prefers a present dbname even when it is null", + entries: vec![ + text("type", "postgres"), + entry("dbname", ProfileValue::Null), + text("database", "db"), + text("pass", "pw"), + text("host", "h"), + ], + expected_result: Ok(NormalizedProfile { + adapter: "postgres", + connection: vec![ + (ConnectionKey::Name("host"), ConnectionSource::Entry(4)), + (ConnectionKey::Name("password"), ConnectionSource::Entry(3)), + ], + target_schema: None, + target_database: None, + warnings: Vec::new(), + }), + }, + NormalizeTestCase { + description: "databricks defaults the catalog", + entries: vec![text("type", "databricks"), text("host", "h"), text("schema", "s")], + expected_result: Ok(NormalizedProfile { + adapter: "databricks", + connection: vec![ + (ConnectionKey::Name("server_hostname"), ConnectionSource::Entry(1)), + (ConnectionKey::Name("catalog"), ConnectionSource::Text("hive_metastore")), + (ConnectionKey::Name("schema"), ConnectionSource::Entry(2)), + ], + target_schema: Some("s".to_owned()), + target_database: Some("hive_metastore".to_owned()), + warnings: Vec::new(), + }), + }, + NormalizeTestCase { + description: "sqlserver rejects windows logins", + entries: vec![text("type", "sqlserver"), entry("windows_login", ProfileValue::Bool(true))], + expected_result: Err(DbtProfileError( + "dbt SQL Server Windows authentication is not supported by SQLBuild dbt-profile normalization yet".to_owned(), + )), + }, + NormalizeTestCase { + description: "missing type", + entries: vec![entry("type", ProfileValue::Other)], + expected_result: Err(DbtProfileError( + "dbt profile type '' is not supported by SQLBuild dbt-profile connection resolution yet. Currently supported: bigquery, databricks, duckdb, postgres, snowflake, sqlserver".to_owned(), + )), + }, + ]; + + for test_case in test_cases { + assert_eq!( + normalize_dbt_profile(&test_case.entries), + test_case.expected_result, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_init_values_when_writing_project_toml_then_keys_and_strings_are_escaped() { + let test_cases = [ + InitTomlTestCase { + description: "plain target with a schema", + config: init_config("dev", Some("main")), + expected_toml: "name = \"analytics\"\nadapter = \"duckdb\"\ndefault_target = \"dev\"\n\n[dbt]\nproject_dir = \"../dbt_project\"\nprofiles_dir = \"C:\\\\profiles\"\ntarget_path = \"../dbt_project/target\"\ntarget = \"dev\"\n\n[connections.dev_dbt_profile]\nsource = \"dbt_profile\"\nprofile = \"ana\\\"lytics\"\ntarget = \"dev\"\n\n[targets.dev]\nconnection = \"dev_dbt_profile\"\nschema = \"main\"\n", + }, + InitTomlTestCase { + description: "dotted target is quoted", + config: init_config("a.b", None), + expected_toml: "name = \"analytics\"\nadapter = \"duckdb\"\ndefault_target = \"a.b\"\n\n[dbt]\nproject_dir = \"../dbt_project\"\nprofiles_dir = \"C:\\\\profiles\"\ntarget_path = \"../dbt_project/target\"\ntarget = \"a.b\"\n\n[connections.\"a.b_dbt_profile\"]\nsource = \"dbt_profile\"\nprofile = \"ana\\\"lytics\"\ntarget = \"a.b\"\n\n[targets.\"a.b\"]\nconnection = \"a.b_dbt_profile\"\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + dbt_init_project_toml(&test_case.config), + test_case.expected_toml, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs b/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs new file mode 100644 index 0000000000..960caa3d8a --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::dbt_profile::errors::DbtProfileError; +use crate::dbt_profile::models::{DbtInitConfig, NormalizedProfile, ProfileEntry}; + +pub(super) struct NormalizeTestCase { + pub(super) description: &'static str, + pub(super) entries: Vec, + pub(super) expected_result: Result, +} + +pub(super) struct InitTomlTestCase { + pub(super) description: &'static str, + pub(super) config: DbtInitConfig, + pub(super) expected_toml: &'static str, +} diff --git a/crates/sqlbuild-config/src/lib.rs b/crates/sqlbuild-config/src/lib.rs index d2ffd151bf..da69b29e36 100644 --- a/crates/sqlbuild-config/src/lib.rs +++ b/crates/sqlbuild-config/src/lib.rs @@ -4,6 +4,8 @@ pub mod command_connection; pub(crate) mod constants; +pub mod dbt_manifest; +pub mod dbt_profile; pub mod effective; pub mod errors; pub mod models; diff --git a/crates/sqlbuild-core/Cargo.toml b/crates/sqlbuild-core/Cargo.toml index c6657c84e0..2d235db809 100644 --- a/crates/sqlbuild-core/Cargo.toml +++ b/crates/sqlbuild-core/Cargo.toml @@ -7,5 +7,8 @@ license.workspace = true publish.workspace = true rust-version.workspace = true +[dependencies] +regex.workspace = true + [lints] workspace = true diff --git a/crates/sqlbuild-core/src/check_command/_helpers/json_payload.rs b/crates/sqlbuild-core/src/check_command/_helpers/json_payload.rs new file mode 100644 index 0000000000..b0cc9bd0bd --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/_helpers/json_payload.rs @@ -0,0 +1,58 @@ +//! JSON payload pieces of check output. + +use crate::check_command::models::{CheckResultText, CheckStatus}; +use crate::json::models::{JsonInteger, JsonValue}; + +/// A JSON string, or null. +pub(crate) fn text(value: Option<&str>) -> JsonValue { + value.map_or(JsonValue::Null, |value| JsonValue::String(value.to_owned())) +} + +/// A JSON integer. +pub(crate) fn integer(value: usize) -> JsonValue { + JsonValue::Integer(JsonInteger::from(i64::try_from(value).unwrap_or(i64::MAX))) +} + +/// A JSON object with entries in order. +pub(crate) fn object(entries: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + entries + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ) +} + +/// The JSON entry of one check. +pub(crate) fn check(result: &CheckResultText) -> JsonValue { + let warnings: JsonValue = if result.warning_messages.is_empty() { + JsonValue::Null + } else { + JsonValue::Array( + result + .warning_messages + .iter() + .map(|warning| JsonValue::String(warning.clone())) + .collect(), + ) + }; + object(vec![ + ("kind", text(Some("python_check"))), + ("name", text(Some(&result.name))), + ("display_name", text(Some(&result.name))), + ( + "check_id", + text(Some(&format!("python_check:{}", result.name))), + ), + ("status", text(Some(result.status.value()))), + ( + "passed", + JsonValue::Bool(result.status == CheckStatus::Pass), + ), + ("severity", text(Some(&result.severity))), + ("message", text(result.message.as_deref())), + ("error_message", text(result.error_message.as_deref())), + ("metadata", result.metadata.clone()), + ("warnings", warnings), + ]) +} diff --git a/crates/sqlbuild-core/src/check_command/_helpers/mod.rs b/crates/sqlbuild-core/src/check_command/_helpers/mod.rs new file mode 100644 index 0000000000..ac8e608de3 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod json_payload; diff --git a/crates/sqlbuild-core/src/check_command/constants.rs b/crates/sqlbuild-core/src/check_command/constants.rs new file mode 100644 index 0000000000..f788cb85bd --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/constants.rs @@ -0,0 +1,8 @@ +//! Check output labels and widths. + +pub(crate) const KIND_LABEL: &str = "check"; +pub(crate) const KIND_WIDTH: usize = 10; +pub(crate) const NAME_WIDTH: usize = 50; +pub(crate) const MULTI_DEPENDENCY_GROUP: &str = "multi-dependency"; +pub(crate) const UNGROUPED: &str = "ungrouped"; +pub(crate) const SELECTION_CODE: &str = "C682"; diff --git a/crates/sqlbuild-core/src/check_command/errors.rs b/crates/sqlbuild-core/src/check_command/errors.rs new file mode 100644 index 0000000000..f166404d9c --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/errors.rs @@ -0,0 +1,8 @@ +//! Check selection errors. + +/// A selection that resolved to Python nodes other than checks. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CheckSelectionError { + pub code: &'static str, + pub message: String, +} diff --git a/crates/sqlbuild-core/src/check_command/main/check_exit_code.rs b/crates/sqlbuild-core/src/check_command/main/check_exit_code.rs new file mode 100644 index 0000000000..21e100fad9 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/check_exit_code.rs @@ -0,0 +1,13 @@ +//! The exit code of `sqb check`. + +use crate::check_command::models::{CheckResultText, CheckStatus}; + +/// 1 when any check failed, else 0. +#[must_use] +pub fn check_exit_code(results: &[CheckResultText]) -> i32 { + i32::from( + results + .iter() + .any(|result| result.status == CheckStatus::Fail), + ) +} diff --git a/crates/sqlbuild-core/src/check_command/main/check_group_header.rs b/crates/sqlbuild-core/src/check_command/main/check_group_header.rs new file mode 100644 index 0000000000..92bd0eaea0 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/check_group_header.rs @@ -0,0 +1,7 @@ +//! The heading of one check group. + +/// `Python checks: ` and a blank line. +#[must_use] +pub fn check_group_header(group: &str) -> String { + format!("Python checks: {group}\n\n") +} diff --git a/crates/sqlbuild-core/src/check_command/main/check_group_label.rs b/crates/sqlbuild-core/src/check_command/main/check_group_label.rs new file mode 100644 index 0000000000..22bb5d89c2 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/check_group_label.rs @@ -0,0 +1,24 @@ +//! The output group of one check. + +use crate::check_command::constants::{MULTI_DEPENDENCY_GROUP, UNGROUPED}; +use crate::check_command::models::CheckGrouping; + +/// The single dependency's name, else the declared group, else `multi-dependency` or `ungrouped`. +#[must_use] +pub fn check_group_label(grouping: &CheckGrouping) -> String { + if let Some(dependencies) = &grouping.dependencies { + if let [only] = dependencies.as_slice() { + return only.clone(); + } + if dependencies.len() > 1 { + return grouping + .group + .clone() + .unwrap_or_else(|| MULTI_DEPENDENCY_GROUP.to_owned()); + } + } + grouping + .group + .clone() + .unwrap_or_else(|| UNGROUPED.to_owned()) +} diff --git a/crates/sqlbuild-core/src/check_command/main/check_json.rs b/crates/sqlbuild-core/src/check_command/main/check_json.rs new file mode 100644 index 0000000000..cbf7b77842 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/check_json.rs @@ -0,0 +1,50 @@ +//! The JSON payload of `sqb check`. + +use crate::check_command::_helpers::json_payload::{check, integer, object, text}; +use crate::check_command::models::{CheckResultText, CheckStatus}; +use crate::json::errors::JsonEmitError; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// Status, counts and checks as `json.dumps(..., indent=2)` with a trailing newline. +pub fn check_json(results: &[CheckResultText]) -> Result { + let count = |status: CheckStatus| { + integer( + results + .iter() + .filter(|result| result.status == status) + .count(), + ) + }; + let failed: bool = results + .iter() + .any(|result| result.status == CheckStatus::Fail); + let payload: JsonValue = object(vec![ + ("version", integer(1)), + ("command", text(Some("check"))), + ( + "status", + text(Some(if failed { "failed" } else { "success" })), + ), + ( + "summary", + object(vec![ + ("pass_count", count(CheckStatus::Pass)), + ("warn_count", count(CheckStatus::Warn)), + ("fail_count", count(CheckStatus::Fail)), + ("total_count", integer(results.len())), + ]), + ), + ( + "checks", + JsonValue::Array(results.iter().map(check).collect()), + ), + ]); + Ok(format!( + "{}\n", + dumps( + &payload, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))) + )? + )) +} diff --git a/crates/sqlbuild-core/src/check_command/main/mod.rs b/crates/sqlbuild-core/src/check_command/main/mod.rs new file mode 100644 index 0000000000..1b40c2cabb --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/mod.rs @@ -0,0 +1,7 @@ +pub mod check_exit_code; +pub mod check_group_header; +pub mod check_group_label; +pub mod check_json; +pub mod render_check_row; +pub mod render_check_summary; +pub mod validate_check_selection; diff --git a/crates/sqlbuild-core/src/check_command/main/render_check_row.rs b/crates/sqlbuild-core/src/check_command/main/render_check_row.rs new file mode 100644 index 0000000000..b3b0241a8c --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/render_check_row.rs @@ -0,0 +1,33 @@ +//! One human check result row. + +use crate::check_command::constants::{KIND_LABEL, KIND_WIDTH, NAME_WIDTH}; +use crate::check_command::models::CheckRow; +use crate::cli_text::models::CliStyle; + +/// Kind, padded name, status, duration and detail, then each warning on its own line. +#[must_use] +pub fn render_check_row(check: &CheckRow, style: CliStyle) -> String { + let result = &check.result; + let padding: String = " ".repeat(NAME_WIDTH.saturating_sub(result.name.chars().count())); + let mut text: String = format!( + " {KIND_LABEL: = result + .error_message + .as_ref() + .filter(|error| !error.is_empty()) + .or(result.message.as_ref()); + if let Some(detail) = detail.filter(|detail| !detail.is_empty()) { + text.push_str(&format!(" {detail}")); + } + text.push('\n'); + for warning in &result.warning_messages { + text.push_str(&format!(" {}\n", style.warning(warning))); + } + text +} diff --git a/crates/sqlbuild-core/src/check_command/main/render_check_summary.rs b/crates/sqlbuild-core/src/check_command/main/render_check_summary.rs new file mode 100644 index 0000000000..5e9d323311 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/render_check_summary.rs @@ -0,0 +1,31 @@ +//! The check summary footer. + +use crate::check_command::models::{CheckResultText, CheckStatus}; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, SummaryCount}; + +fn count(results: &[CheckResultText], status: Option) -> i64 { + let matched: usize = results + .iter() + .filter(|result| status.is_none_or(|status| result.status == status)) + .count(); + i64::try_from(matched).unwrap_or(i64::MAX) +} + +/// A blank line and the PASS, WARN, FAIL and TOTAL counts. +#[must_use] +pub fn render_check_summary(results: &[CheckResultText], style: CliStyle) -> String { + let counts: Vec = [ + ("PASS", Some(CheckStatus::Pass)), + ("WARN", Some(CheckStatus::Warn)), + ("FAIL", Some(CheckStatus::Fail)), + ("TOTAL", None), + ] + .into_iter() + .map(|(label, status)| SummaryCount { + label: label.to_owned(), + value: count(results, status), + }) + .collect(); + format!("\n{}\n", format_summary_footer(style, &counts, None)) +} diff --git a/crates/sqlbuild-core/src/check_command/main/validate_check_selection.rs b/crates/sqlbuild-core/src/check_command/main/validate_check_selection.rs new file mode 100644 index 0000000000..3a47bea587 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/main/validate_check_selection.rs @@ -0,0 +1,20 @@ +//! Refuse a check selection that resolved to Python nodes other than checks. + +use crate::check_command::constants::SELECTION_CODE; +use crate::check_command::errors::CheckSelectionError; + +/// C682 naming the sorted non-check selections, when there are any. +pub fn validate_check_selection(non_check_names: &[String]) -> Result<(), CheckSelectionError> { + if non_check_names.is_empty() { + return Ok(()); + } + let mut names: Vec<&str> = non_check_names.iter().map(String::as_str).collect(); + names.sort_unstable(); + Err(CheckSelectionError { + code: SELECTION_CODE, + message: format!( + "sqb check selectors must resolve only to Python checks; non-check selections: {}", + names.join(", ") + ), + }) +} diff --git a/crates/sqlbuild-core/src/check_command/mod.rs b/crates/sqlbuild-core/src/check_command/mod.rs new file mode 100644 index 0000000000..85a89a753f --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/mod.rs @@ -0,0 +1,10 @@ +//! `sqb check`: check selection errors, grouped result rows, JSON payload and the exit code. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/check_command/models.rs b/crates/sqlbuild-core/src/check_command/models.rs new file mode 100644 index 0000000000..6214d0a615 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/models.rs @@ -0,0 +1,61 @@ +//! Python check results as the command renders them. + +use crate::json::models::JsonValue; + +/// How one check ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CheckStatus { + Pass, + Warn, + Fail, +} + +impl CheckStatus { + /// The label rows and the summary use. + #[must_use] + pub fn label(self) -> &'static str { + match self { + Self::Pass => "PASS", + Self::Warn => "WARN", + Self::Fail => "FAIL", + } + } + + /// The status JSON output spells. + #[must_use] + pub fn value(self) -> &'static str { + match self { + Self::Pass => "pass", + Self::Warn => "warn", + Self::Fail => "fail", + } + } +} + +/// One executed Python check. +#[derive(Clone, Debug, PartialEq)] +pub struct CheckResultText { + pub name: String, + pub status: CheckStatus, + /// The `CheckSeverity` value. + pub severity: String, + pub message: Option, + pub error_message: Option, + pub metadata: JsonValue, + pub warning_messages: Vec, +} + +/// What decides a check's output group: its dependencies and declared group. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CheckGrouping { + /// Python dependency names, then SQL dependency names; `None` without a Python graph. + pub dependencies: Option>, + pub group: Option, +} + +/// One rendered row: the result and the duration the progress projector recorded. +#[derive(Clone, Debug, PartialEq)] +pub struct CheckRow { + pub result: CheckResultText, + pub duration_ms: Option, +} diff --git a/crates/sqlbuild-core/src/check_command/tests.rs b/crates/sqlbuild-core/src/check_command/tests.rs new file mode 100644 index 0000000000..3665959f9c --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_check_command.rs"] +mod check_command; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/check_command/tests/helpers.rs b/crates/sqlbuild-core/src/check_command/tests/helpers.rs new file mode 100644 index 0000000000..892194ecf9 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/tests/helpers.rs @@ -0,0 +1,24 @@ +use crate::check_command::models::{CheckGrouping, CheckResultText, CheckStatus}; +use crate::json::models::JsonValue; + +/// A check result with no message, metadata or warnings. +pub(super) fn result(name: &str, status: CheckStatus) -> CheckResultText { + CheckResultText { + name: name.to_owned(), + status, + severity: "error".to_owned(), + message: None, + error_message: None, + metadata: JsonValue::Object(Vec::new()), + warning_messages: Vec::new(), + } +} + +/// A grouping with dependencies and a declared group. +pub(super) fn grouping(dependencies: Option<&[&str]>, group: Option<&str>) -> CheckGrouping { + CheckGrouping { + dependencies: dependencies + .map(|names| names.iter().map(|name| (*name).to_owned()).collect()), + group: group.map(str::to_owned), + } +} diff --git a/crates/sqlbuild-core/src/check_command/tests/test_check_command.rs b/crates/sqlbuild-core/src/check_command/tests/test_check_command.rs new file mode 100644 index 0000000000..f28fd941a5 --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/tests/test_check_command.rs @@ -0,0 +1,190 @@ +use crate::check_command::main::check_exit_code::check_exit_code; +use crate::check_command::main::check_group_label::check_group_label; +use crate::check_command::main::check_json::check_json; +use crate::check_command::main::render_check_row::render_check_row; +use crate::check_command::main::render_check_summary::render_check_summary; +use crate::check_command::main::validate_check_selection::validate_check_selection; +use crate::check_command::models::{CheckResultText, CheckRow, CheckStatus}; +use crate::check_command::tests::helpers::{grouping, result}; +use crate::check_command::tests::test_types::{ + GroupTestCase, RowTestCase, SelectionTestCase, SummaryTestCase, +}; +use crate::cli_text::models::CliStyle; +use crate::json::models::{JsonInteger, JsonValue}; + +#[test] +fn given_dependencies_when_grouping_then_labels_by_single_dependency_or_group() { + let test_cases = [ + GroupTestCase { + description: "one dependency names the group", + grouping: grouping(Some(&["export_orders"]), Some("declared")), + expected_label: "export_orders", + }, + GroupTestCase { + description: "several dependencies use the declared group", + grouping: grouping(Some(&["a", "b"]), Some("declared")), + expected_label: "declared", + }, + GroupTestCase { + description: "several dependencies without a group are multi-dependency", + grouping: grouping(Some(&["a", "b"]), None), + expected_label: "multi-dependency", + }, + GroupTestCase { + description: "no graph and no group is ungrouped", + grouping: grouping(None, None), + expected_label: "ungrouped", + }, + ]; + + for test_case in test_cases { + assert_eq!( + check_group_label(&test_case.grouping), + test_case.expected_label, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_row_when_rendering_then_pads_name_and_shows_detail() { + let test_cases = [ + RowTestCase { + description: "an error message wins over the message and warnings follow", + row: CheckRow { + result: CheckResultText { + message: Some("ignored".to_owned()), + error_message: Some("boom".to_owned()), + warning_messages: vec!["careful".to_owned()], + ..result("orders", CheckStatus::Fail) + }, + duration_ms: Some(125.0), + }, + use_color: false, + expected_text: " check orders FAIL 0.12s boom\n careful\n", + }, + RowTestCase { + description: "colour styles the status and an empty error falls back to the message", + row: CheckRow { + result: CheckResultText { + message: Some("fine".to_owned()), + error_message: Some(String::new()), + ..result("orders", CheckStatus::Pass) + }, + duration_ms: None, + }, + use_color: true, + expected_text: " check orders \u{1b}[32mPASS\u{1b}[0m fine\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_check_row( + &test_case.row, + CliStyle { + use_color: test_case.use_color + } + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_summarizing_then_counts_serializes_and_exits() { + let test_cases = [ + SummaryTestCase { + description: "a warning does not fail the run", + results: vec![ + CheckResultText { + metadata: JsonValue::Object(vec![( + "rows".to_owned(), + JsonValue::Integer(JsonInteger::from(3_i64)), + )]), + ..result("a", CheckStatus::Pass) + }, + CheckResultText { + warning_messages: vec!["w".to_owned()], + severity: "warn".to_owned(), + ..result("b", CheckStatus::Warn) + }, + ], + expected_summary: "\nPASS=1 WARN=1 FAIL=0 TOTAL=2\n", + expected_json: "{\n \"version\": 1,\n \"command\": \"check\",\n \"status\": \"success\",\n \ + \"summary\": {\n \"pass_count\": 1,\n \"warn_count\": 1,\n \"fail_count\": 0,\n \ + \"total_count\": 2\n },\n \"checks\": [\n {\n \"kind\": \"python_check\",\n \ + \"name\": \"a\",\n \"display_name\": \"a\",\n \"check_id\": \"python_check:a\",\n \ + \"status\": \"pass\",\n \"passed\": true,\n \"severity\": \"error\",\n \ + \"message\": null,\n \"error_message\": null,\n \"metadata\": {\n \ + \"rows\": 3\n },\n \"warnings\": null\n },\n {\n \ + \"kind\": \"python_check\",\n \"name\": \"b\",\n \"display_name\": \"b\",\n \ + \"check_id\": \"python_check:b\",\n \"status\": \"warn\",\n \"passed\": false,\n \ + \"severity\": \"warn\",\n \"message\": null,\n \"error_message\": null,\n \ + \"metadata\": {},\n \"warnings\": [\n \"w\"\n ]\n }\n ]\n}\n", + expected_exit_code: 0, + }, + SummaryTestCase { + description: "a failure fails the run", + results: vec![result("c", CheckStatus::Fail)], + expected_summary: "\nPASS=0 WARN=0 FAIL=1 TOTAL=1\n", + expected_json: "{\n \"version\": 1,\n \"command\": \"check\",\n \"status\": \"failed\",\n \ + \"summary\": {\n \"pass_count\": 0,\n \"warn_count\": 0,\n \"fail_count\": 1,\n \ + \"total_count\": 1\n },\n \"checks\": [\n {\n \"kind\": \"python_check\",\n \ + \"name\": \"c\",\n \"display_name\": \"c\",\n \"check_id\": \"python_check:c\",\n \ + \"status\": \"fail\",\n \"passed\": false,\n \"severity\": \"error\",\n \ + \"message\": null,\n \"error_message\": null,\n \"metadata\": {},\n \ + \"warnings\": null\n }\n ]\n}\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + render_check_summary(&test_case.results, CliStyle { use_color: false }), + check_json(&test_case.results).ok(), + check_exit_code(&test_case.results), + ), + ( + test_case.expected_summary.to_owned(), + Some(test_case.expected_json.to_owned()), + test_case.expected_exit_code, + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selected_names_when_validating_then_rejects_non_checks() { + let test_cases = [ + SelectionTestCase { + description: "only checks pass", + non_checks: Vec::new(), + expected_message: None, + }, + SelectionTestCase { + description: "non-check names are listed sorted", + non_checks: vec!["task_b".to_owned(), "asset_a".to_owned()], + expected_message: Some( + "sqb check selectors must resolve only to Python checks; non-check selections: asset_a, task_b", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + validate_check_selection(&test_case.non_checks) + .err() + .map(|error| error.message), + test_case.expected_message.map(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/check_command/tests/test_types.rs b/crates/sqlbuild-core/src/check_command/tests/test_types.rs new file mode 100644 index 0000000000..967c8d64ab --- /dev/null +++ b/crates/sqlbuild-core/src/check_command/tests/test_types.rs @@ -0,0 +1,28 @@ +use crate::check_command::models::{CheckGrouping, CheckResultText, CheckRow}; + +pub(super) struct GroupTestCase { + pub(super) description: &'static str, + pub(super) grouping: CheckGrouping, + pub(super) expected_label: &'static str, +} + +pub(super) struct RowTestCase { + pub(super) description: &'static str, + pub(super) row: CheckRow, + pub(super) use_color: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct SummaryTestCase { + pub(super) description: &'static str, + pub(super) results: Vec, + pub(super) expected_summary: &'static str, + pub(super) expected_json: &'static str, + pub(super) expected_exit_code: i32, +} + +pub(super) struct SelectionTestCase { + pub(super) description: &'static str, + pub(super) non_checks: Vec, + pub(super) expected_message: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs b/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs new file mode 100644 index 0000000000..b80362ec49 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs @@ -0,0 +1,31 @@ +//! A result-stored coded error, as CLI expected errors render it. + +use crate::cli_text::models::CliStyle; + +/// `error[CODE]: message` or `[CODE] message`, later lines muted, then any help line. +pub(crate) fn coded_error( + (code, message, help): (&str, &str, Option<&str>), + include_error_label: bool, + style: CliStyle, +) -> String { + let mut lines = message.split('\n'); + let first: &str = lines.next().unwrap_or(""); + let rest: Vec = lines.map(|line| style.muted(line)).collect(); + let rendered_message: String = if rest.is_empty() { + message.to_owned() + } else { + format!("{first}\n{}", rest.join("\n")) + }; + let mut rendered: String = format!( + "{} {rendered_message}", + style.error_strong(&if include_error_label { + format!("error[{code}]:") + } else { + format!("[{code}]") + }) + ); + if let Some(help) = help { + rendered = format!("{rendered}\n {} {help}", style.muted("= help:")); + } + rendered +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan.rs b/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan.rs new file mode 100644 index 0000000000..9a02df2c7e --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan.rs @@ -0,0 +1,197 @@ +//! Sections of the `sqb dbt` plan text. + +use std::collections::BTreeMap; + +use crate::cli_text::_helpers::layout::tree_connector; +use crate::cli_text::_helpers::summary_text::overflow_line; +use crate::cli_text::constants::DBT_RESOURCE_FALLBACK_TYPE; +use crate::cli_text::models::{CliStyle, DbtInteropPlanText, DbtLsNodeText, DisplayLimit}; +use crate::text::main::python_splitlines::python_splitlines; + +const STYLE: CliStyle = CliStyle { use_color: true }; + +/// `visible_entries`: the entries shown under the display cap. +fn visible(entries: &[T], display: &DisplayLimit) -> usize { + display + .max_entries + .map_or(entries.len(), |limit| limit.min(entries.len())) +} + +fn display_name(node: &DbtLsNodeText) -> String { + if !node.fqn.is_empty() { + return node.fqn.join("."); + } + node.name + .clone() + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| node.unique_id.clone()) +} + +fn node_lines(nodes: &[&DbtLsNodeText], display: &DisplayLimit) -> Vec { + let shown: usize = visible(nodes, display); + let mut lines: Vec = Vec::new(); + for (index, node) in nodes[..shown].iter().enumerate() { + let last: bool = index + 1 == shown && nodes.len() <= shown; + lines.push(format!( + " {} {}", + tree_connector(STYLE, last), + STYLE.object_name(&display_name(node)) + )); + } + let hidden: i64 = i64::try_from(nodes.len() - shown).unwrap_or(i64::MAX); + lines.extend(overflow_line(hidden, " ", &display.overflow_flag)); + lines +} + +/// The dbt section: counts, command, selected nodes by resource type and required ids. +pub(crate) fn dbt_section(plan: &DbtInteropPlanText, display: &DisplayLimit) -> Vec { + let required: &[String] = &plan.dbt_required_unique_ids; + let mut lines: Vec = vec![format!( + "{} {}", + STYLE.dbt_label("dbt"), + STYLE.dbt_section(&format!( + "({} selected, {} required)", + plan.dbt_selected_unique_ids.len(), + required.len() + )) + )]; + if plan.dbt_skip_reason.is_some() { + lines.push(STYLE.muted(" skipped: no dbt work selected")); + return lines; + } + lines.push(format!( + " {}: {}", + STYLE.label("command"), + STYLE.command(&plan.dbt_argv.join(" ")) + )); + let mut sorted: Vec<(String, String, &DbtLsNodeText)> = plan + .dbt_nodes + .iter() + .map(|node| { + ( + node.resource_type.clone().unwrap_or_default(), + display_name(node), + node, + ) + }) + .collect(); + sorted.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + let mut by_type: BTreeMap> = BTreeMap::new(); + for (_, _, node) in sorted { + let resource_type: String = node + .resource_type + .clone() + .filter(|kind| !kind.is_empty()) + .unwrap_or_else(|| DBT_RESOURCE_FALLBACK_TYPE.to_owned()); + by_type.entry(resource_type).or_default().push(node); + } + for (resource_type, nodes) in &by_type { + lines.push(String::new()); + lines.push(format!( + " {}", + STYLE.plan_section(&format!("{resource_type} ({})", nodes.len())) + )); + lines.extend(node_lines(nodes, display)); + } + if !required.is_empty() { + lines.push(String::new()); + lines.push(STYLE.dbt_label(&format!(" required ({})", required.len()))); + let shown: usize = visible(required, display); + for (index, unique_id) in required[..shown].iter().enumerate() { + lines.push(format!( + " {} {}", + tree_connector(STYLE, index + 1 == shown), + STYLE.object_name(unique_id) + )); + } + } + lines +} + +/// The SQLBuild section: the rendered SQLBuild plan, or the selected model names. +pub(crate) fn sqlbuild_section( + plan: &DbtInteropPlanText, + sqlbuild_plan: Option<&str>, + display: &DisplayLimit, +) -> Vec { + let mut lines: Vec = + vec![STYLE.plan_section(&format!("SQLBuild ({})", plan.sqlbuild_models.len()))]; + if plan.sqlbuild_skip_reason.is_some() { + lines.push(STYLE.muted(" skipped: no SQLBuild work selected")); + return lines; + } + if let Some(rendered) = sqlbuild_plan { + lines.extend(python_splitlines(rendered).into_iter().map(|line| { + if line.is_empty() { + String::new() + } else { + format!(" {line}") + } + })); + return lines; + } + let shown: usize = visible(&plan.sqlbuild_models, display); + lines.extend( + plan.sqlbuild_models[..shown] + .iter() + .map(|name| format!(" {}", STYLE.object_name(name))), + ); + lines +} + +/// Anchor terms, path translations and warnings, each after a blank line when present. +pub(crate) fn trailing_sections(plan: &DbtInteropPlanText, display: &DisplayLimit) -> Vec { + let mut lines: Vec = Vec::new(); + if !plan.anchors.is_empty() { + lines.push(String::new()); + lines.push(STYLE.label("dbt selector anchors")); + let shown: usize = visible(&plan.anchors, display); + for (term, unique_ids) in &plan.anchors[..shown] { + lines.push(format!( + " {}: {} dbt resources", + STYLE.command(term), + unique_ids.len() + )); + } + } + if !plan.path_translations.is_empty() { + lines.push(String::new()); + lines.push(STYLE.label("path translations")); + for (original, translated) in &plan.path_translations { + lines.push(format!( + " {} -> {}", + STYLE.command(original), + STYLE.command(translated) + )); + } + } + if !plan.warnings.is_empty() { + lines.push(String::new()); + lines.push(STYLE.warning("Warnings")); + for warning in &plan.warnings { + lines.push(STYLE.warning(&format!(" {warning}"))); + } + } + lines +} + +/// The text with every `ESC[...m` color sequence removed. +pub(crate) fn without_ansi(text: &str) -> String { + let mut result: String = String::with_capacity(text.len()); + let mut rest: &str = text; + while let Some(start) = rest.find("\x1b[") { + result.push_str(&rest[..start]); + let after: &str = &rest[start + 2..]; + let digits: usize = after + .find(|character: char| !(character.is_ascii_digit() || character == ';')) + .unwrap_or(after.len()); + if after[digits..].starts_with('m') { + rest = &after[digits + 1..]; + } else { + result.push_str("\x1b["); + rest = after; + } + } + result.push_str(rest); + result +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan_json.rs b/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan_json.rs new file mode 100644 index 0000000000..6ca1a9a0a2 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/dbt_plan_json.rs @@ -0,0 +1,92 @@ +//! The JSON payload of a `sqb dbt` plan. + +use crate::cli_text::models::DbtInteropPlanText; +use crate::json::models::JsonValue; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn texts(values: &[String]) -> JsonValue { + JsonValue::Array(values.iter().map(|value| text(value)).collect()) +} + +fn argvs(values: &[Vec]) -> JsonValue { + JsonValue::Array(values.iter().map(|argv| texts(argv)).collect()) +} + +fn object(fields: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + fields + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + ) +} + +fn skip_reason(reason: Option<&String>) -> JsonValue { + reason.map_or(JsonValue::Null, |reason| text(reason)) +} + +/// The plan's dbt and SQLBuild work as the JSON payload Python built. +pub(crate) fn dbt_plan_payload(plan: &DbtInteropPlanText) -> JsonValue { + object(vec![ + ("command", text(&plan.command)), + ( + "dbt", + object(vec![ + ("argv", texts(&plan.dbt_argv)), + ("supplemental_argvs", argvs(&plan.supplemental_dbt_argvs)), + ("selected_unique_ids", texts(&plan.dbt_selected_unique_ids)), + ("required_unique_ids", texts(&plan.dbt_required_unique_ids)), + ( + "required_selector_terms", + texts(&plan.dbt_required_selector_terms), + ), + ("skipped", JsonValue::Bool(plan.dbt_skip_reason.is_some())), + ("skip_reason", skip_reason(plan.dbt_skip_reason.as_ref())), + ]), + ), + ( + "sqlbuild", + object(vec![ + ("argvs", argvs(&plan.sqlbuild_argvs)), + ("selected_models", texts(&plan.sqlbuild_models)), + ( + "skipped", + JsonValue::Bool(plan.sqlbuild_skip_reason.is_some()), + ), + ( + "skip_reason", + skip_reason(plan.sqlbuild_skip_reason.as_ref()), + ), + ]), + ), + ( + "anchors", + JsonValue::Array( + plan.anchors + .iter() + .map(|(term, unique_ids)| { + object(vec![ + ("term", text(term)), + ("unique_ids", texts(unique_ids)), + ]) + }) + .collect(), + ), + ), + ( + "path_translations", + JsonValue::Array( + plan.path_translations + .iter() + .map(|(original, translated)| { + object(vec![("from", text(original)), ("to", text(translated))]) + }) + .collect(), + ), + ), + ("warnings", texts(&plan.warnings)), + ]) +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs index f450e37ec1..34f8daf1d2 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs @@ -1,4 +1,7 @@ pub(crate) mod bounded_sections; +pub(crate) mod coded_error; +pub(crate) mod dbt_plan; +pub(crate) mod dbt_plan_json; pub(crate) mod diagnostics; pub(crate) mod layout; pub(crate) mod plan_document; diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index eae5ddd0a4..6e575a762f 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -6,6 +6,7 @@ pub(crate) const RED: &str = "\x1b[38;5;167m"; pub(crate) const GREEN: &str = "\x1b[32m"; pub(crate) const YELLOW: &str = "\x1b[33m"; pub(crate) const BLUE: &str = "\x1b[34m"; +pub(crate) const ORANGE: &str = "\x1b[38;5;208m"; pub(crate) const RESET: &str = "\x1b[0m"; pub(crate) const SUCCESS_STATUS_LABELS: [&str; 3] = ["OK", "PASS", "SUCCESS"]; pub(crate) const WARNING_STATUS_LABELS: [&str; 2] = ["WARN", "WARNING"]; @@ -84,3 +85,5 @@ pub(crate) const FOOTER_ERROR_LABELS: [&str; 4] = ["FAIL", "ERROR", "SYNC_FAIL", pub(crate) const FOOTER_SKIP_LABELS: [&str; 1] = ["SKIP"]; pub(crate) const COUNT_HEADER_OPENING: &str = " ("; pub(crate) const COUNT_HEADER_CLOSING: &str = ")"; +pub(crate) const DBT_NO_MATCH_WARNING: &str = "No dbt or SQLBuild resources matched the selection."; +pub(crate) const DBT_RESOURCE_FALLBACK_TYPE: &str = "resource"; diff --git a/crates/sqlbuild-core/src/cli_text/main/dbt_interop_plan_warnings.rs b/crates/sqlbuild-core/src/cli_text/main/dbt_interop_plan_warnings.rs new file mode 100644 index 0000000000..a5cd9572e1 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/dbt_interop_plan_warnings.rs @@ -0,0 +1,18 @@ +//! The warnings a `sqb dbt` plan reports. + +use crate::cli_text::constants::DBT_NO_MATCH_WARNING; + +/// `warnings`, plus the no-match warning when neither dbt nor SQLBuild has work. +#[must_use] +pub fn dbt_interop_plan_warnings(warnings: Vec, has_work: bool) -> Vec { + if has_work + || warnings + .iter() + .any(|warning| warning == DBT_NO_MATCH_WARNING) + { + return warnings; + } + let mut warnings: Vec = warnings; + warnings.push(DBT_NO_MATCH_WARNING.to_owned()); + warnings +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_coded_error.rs b/crates/sqlbuild-core/src/cli_text/main/format_coded_error.rs new file mode 100644 index 0000000000..092b0b4b39 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_coded_error.rs @@ -0,0 +1,14 @@ +//! A result-stored coded error, as CLI expected errors render it. + +use crate::cli_text::_helpers::coded_error::coded_error; +use crate::cli_text::models::CliStyle; + +/// `format_coded_error`: the coded message, later lines muted, then any help line. +#[must_use] +pub fn format_coded_error( + (code, message, help): (&str, &str, Option<&str>), + include_error_label: bool, + style: CliStyle, +) -> String { + coded_error((code, message, help), include_error_label, style) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan.rs b/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan.rs new file mode 100644 index 0000000000..818b39267a --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan.rs @@ -0,0 +1,38 @@ +//! The human `sqb dbt` plan. + +use crate::cli_text::_helpers::dbt_plan::{ + dbt_section, sqlbuild_section, trailing_sections, without_ansi, +}; +use crate::cli_text::_helpers::layout::surface_header; +use crate::cli_text::models::{CliStyle, DbtInteropPlanText, DisplayLimit}; + +/// Selected dbt and SQLBuild work, colored or plain; `sqlbuild_plan` is the rendered SQLBuild plan. +#[must_use] +pub fn format_dbt_interop_plan( + plan: &DbtInteropPlanText, + sqlbuild_plan: Option<&str>, + (display, use_color): (&DisplayLimit, bool), +) -> String { + let mut dbt_ids: Vec<&String> = plan + .dbt_selected_unique_ids + .iter() + .chain(&plan.dbt_required_unique_ids) + .collect(); + dbt_ids.sort_unstable(); + dbt_ids.dedup(); + let selected: usize = dbt_ids.len() + plan.sqlbuild_models.len(); + let mut lines: Vec = vec![ + surface_header( + CliStyle { use_color: true }, + "Plan ready", + &format!("{selected} selected resources"), + ), + String::new(), + ]; + lines.extend(dbt_section(plan, display)); + lines.push(String::new()); + lines.extend(sqlbuild_section(plan, sqlbuild_plan, display)); + lines.extend(trailing_sections(plan, display)); + let text: String = lines.join("\n"); + if use_color { text } else { without_ansi(&text) } +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan_json.rs b/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan_json.rs new file mode 100644 index 0000000000..12915a64b3 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_dbt_interop_plan_json.rs @@ -0,0 +1,17 @@ +//! `sqb dbt plan --json`. + +use crate::cli_text::_helpers::dbt_plan_json::dbt_plan_payload; +use crate::cli_text::models::DbtInteropPlanText; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// `json.dumps(payload, indent=2)` of the plan's dbt and SQLBuild work. +#[must_use] +pub fn format_dbt_interop_plan_json(plan: &DbtInteropPlanText) -> String { + let payload: JsonValue = dbt_plan_payload(plan); + dumps( + &payload, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + ) + .unwrap_or_default() +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_tree_connector.rs b/crates/sqlbuild-core/src/cli_text/main/format_tree_connector.rs new file mode 100644 index 0000000000..423012e01f --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_tree_connector.rs @@ -0,0 +1,10 @@ +//! The dim connector of one tree row. + +use crate::cli_text::_helpers::layout::tree_connector; +use crate::cli_text::models::CliStyle; + +/// `tree_connector`: a dim branch, or the last branch of a group. +#[must_use] +pub fn format_tree_connector(style: CliStyle, last: bool) -> String { + tree_connector(style, last) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/mod.rs b/crates/sqlbuild-core/src/cli_text/main/mod.rs index f64b0962c9..8208efb9bd 100644 --- a/crates/sqlbuild-core/src/cli_text/main/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/main/mod.rs @@ -1,7 +1,11 @@ +pub mod dbt_interop_plan_warnings; pub mod format_aligned_name_value; +pub mod format_coded_error; pub mod format_compile_text; pub mod format_completion_line; pub mod format_count_noun; +pub mod format_dbt_interop_plan; +pub mod format_dbt_interop_plan_json; pub mod format_overflow_line; pub mod format_phase_line; pub mod format_plan_completion; @@ -9,6 +13,7 @@ pub mod format_plan_text; pub mod format_status_cell; pub mod format_summary_footer; pub mod format_surface_header; +pub mod format_tree_connector; pub mod resolve_name_column_width; pub mod semantic_coverage_notice; pub mod split_count_header; diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index ac0d0a03b0..4ff7b7d80e 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -1,7 +1,7 @@ //! Plain data of the human compile report, and the semantic styling it is rendered with. use crate::cli_text::constants::{ - BLUE, BOLD, DIM, ERROR_STATUS_LABELS, GREEN, RED, RESET, SKIPPED_STATUS_LABELS, + BLUE, BOLD, DIM, ERROR_STATUS_LABELS, GREEN, ORANGE, RED, RESET, SKIPPED_STATUS_LABELS, SUCCESS_STATUS_LABELS, WARNING_STATUS_LABELS, YELLOW, }; @@ -70,6 +70,16 @@ impl CliStyle { self.apply("", text) } + #[must_use] + pub fn dbt_label(self, text: &str) -> String { + self.apply(ORANGE, text) + } + + #[must_use] + pub fn dbt_section(self, text: &str) -> String { + self.apply(BOLD, text) + } + #[must_use] pub fn success(self, text: &str) -> String { self.apply(GREEN, text) @@ -506,3 +516,34 @@ impl CliDocument { rendered } } + +/// One node `dbt ls` selected. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtLsNodeText { + pub unique_id: String, + pub resource_type: Option, + pub name: Option, + pub fqn: Vec, +} + +/// Everything a `sqb dbt` plan shows, in text and JSON. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtInteropPlanText { + /// `DbtInteropCommand` value. + pub command: String, + pub dbt_argv: Vec, + pub supplemental_dbt_argvs: Vec>, + pub dbt_nodes: Vec, + pub dbt_selected_unique_ids: Vec, + pub dbt_required_unique_ids: Vec, + pub dbt_required_selector_terms: Vec, + /// `DbtInteropSkipReason` value, when dbt is skipped. + pub dbt_skip_reason: Option, + pub sqlbuild_argvs: Vec>, + pub sqlbuild_models: Vec, + pub sqlbuild_skip_reason: Option, + /// Each anchor term with the dbt unique ids it matched. + pub anchors: Vec<(String, Vec)>, + pub path_translations: Vec<(String, String)>, + pub warnings: Vec, +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index e1b88d9e36..ddac7563c2 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -1,15 +1,17 @@ use crate::cli_text::_helpers::diagnostics::diagnostics_text; use crate::cli_text::main::format_compile_text::format_compile_text; +use crate::cli_text::main::format_dbt_interop_plan::format_dbt_interop_plan; use crate::cli_text::main::format_plan_completion::format_plan_completion; use crate::cli_text::main::format_plan_text::format_plan_text; use crate::cli_text::main::semantic_coverage_notice::semantic_coverage_notice; use crate::cli_text::models::{ - CliStyle, CompileTextReport, DiagnosticText, LineageNote, ModelLine, ModelPlanText, - PlanTextReport, PlanWarningText, RelatedText, + CliStyle, CompileTextReport, DbtInteropPlanText, DbtLsNodeText, DiagnosticText, DisplayLimit, + LineageNote, ModelLine, ModelPlanText, PlanTextReport, PlanWarningText, RelatedText, }; use crate::cli_text::tests::helpers::{diagnostic, location, plan_report, table_model}; use crate::cli_text::tests::test_types::{ - CompileTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, PlanTextTestCase, + CompileTextTestCase, DbtPlanTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, + PlanTextTestCase, }; /// Expectations recorded from the retired Python formatter. @@ -182,3 +184,51 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { ); } } + +/// Expectations recorded from the retired Python formatter. +#[test] +fn given_dbt_interop_plan_when_formatting_then_matches_python_text() { + let node = |unique_id: &str, resource_type: Option<&str>| DbtLsNodeText { + unique_id: unique_id.to_owned(), + resource_type: resource_type.map(str::to_owned), + name: None, + fqn: Vec::new(), + }; + let test_cases = [DbtPlanTextTestCase { + description: "grouped nodes with overflow, required ids, anchors and warnings", + plan: DbtInteropPlanText { + command: "plan".to_owned(), + dbt_argv: vec!["dbt".to_owned(), "ls".to_owned()], + dbt_nodes: vec![ + node("model.a.b", Some("model")), + node("model.a.a", Some("model")), + node("x", None), + ], + dbt_selected_unique_ids: vec![ + "model.a.a".to_owned(), + "model.a.b".to_owned(), + "x".to_owned(), + ], + dbt_required_unique_ids: vec!["model.a.r".to_owned()], + sqlbuild_models: vec!["orders".to_owned()], + anchors: vec![("a+".to_owned(), vec!["u".to_owned()])], + warnings: vec!["careful".to_owned()], + ..DbtInteropPlanText::default() + }, + max_entries: Some(1), + expected_text: "Plan ready 5 selected resources\n\ndbt (3 selected, 1 required)\n command: dbt ls\n\n model (2)\n ├── model.a.a\n ... and 1 more (use --verbose to show all)\n\n resource (1)\n └── x\n\n required (1)\n └── model.a.r\n\nSQLBuild (1)\n orders\n\ndbt selector anchors\n a+: 1 dbt resources\n\nWarnings\n careful", + }]; + + for test_case in test_cases { + let display: DisplayLimit = DisplayLimit { + max_entries: test_case.max_entries, + overflow_flag: "--verbose".to_owned(), + }; + assert_eq!( + format_dbt_interop_plan(&test_case.plan, None, (&display, false)), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs index 09c3dd0dda..ed807492b8 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs @@ -25,3 +25,10 @@ pub(super) struct PlanTextTestCase { pub(super) expected_text: &'static str, pub(super) expected_completion: &'static str, } + +pub(super) struct DbtPlanTextTestCase { + pub(super) description: &'static str, + pub(super) plan: crate::cli_text::models::DbtInteropPlanText, + pub(super) max_entries: Option, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/clone_text/_helpers/mod.rs b/crates/sqlbuild-core/src/clone_text/_helpers/mod.rs new file mode 100644 index 0000000000..9446b134c0 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod quoted; diff --git a/crates/sqlbuild-core/src/clone_text/_helpers/quoted.rs b/crates/sqlbuild-core/src/clone_text/_helpers/quoted.rs new file mode 100644 index 0000000000..25f981ae02 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/_helpers/quoted.rs @@ -0,0 +1,57 @@ +//! Quoted identifier spans in a relation name. + +/// The end of a doubled-escape quoted identifier opening at `start`, as a backtracking regex finds it. +fn quoted_end(chars: &[char], start: usize) -> Option { + let close: char = match chars[start] { + '"' => '"', + '`' => '`', + '[' => ']', + _ => return None, + }; + let mut position: usize = start + 1; + let mut last_pair: Option = None; + while position < chars.len() { + if chars[position] != close { + position += 1; + continue; + } + if chars.get(position + 1) == Some(&close) { + last_pair = Some(position); + position += 2; + continue; + } + return Some(position + 1); + } + last_pair.map(|pair| pair + 1) +} + +/// The relation split into `(text, quoted)` segments in order. +fn quoted_segments(relation: &str) -> Vec<(String, bool)> { + let chars: Vec = relation.chars().collect(); + let mut segments: Vec<(String, bool)> = Vec::new(); + let mut plain: String = String::new(); + let mut position: usize = 0; + while position < chars.len() { + match quoted_end(&chars, position) { + Some(end) => { + segments.push((std::mem::take(&mut plain), false)); + segments.push((chars[position..end].iter().collect(), true)); + position = end; + } + None => { + plain.push(chars[position]); + position += 1; + } + } + } + segments.push((plain, false)); + segments +} + +/// Lowercase ordinary identifiers while preserving quoted identifier case. +pub(crate) fn clone_relation_display_name(relation: &str) -> String { + quoted_segments(relation) + .into_iter() + .map(|(text, quoted)| if quoted { text } else { text.to_lowercase() }) + .collect() +} diff --git a/crates/sqlbuild-core/src/clone_text/constants.rs b/crates/sqlbuild-core/src/clone_text/constants.rs new file mode 100644 index 0000000000..9a30eab26f --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/constants.rs @@ -0,0 +1,24 @@ +//! Clone output labels. + +use crate::clone_text::models::CloneStatusKind; + +/// The width of the longest clone action value. +pub(crate) const ACTION_WIDTH: usize = "skipped_missing_dependency".len(); +pub(crate) const FINGERPRINT_LABEL: &str = "fingerprint"; +pub(crate) const FINGERPRINT_LABEL_WIDTH: usize = 10; +pub(crate) const PENDING_IDENTITY_LIMIT: usize = 10; + +/// Action footer labels and the action value each counts. +pub(crate) const ACTION_SUMMARY: [(&str, &str); 4] = [ + ("CLONED", "cloned"), + ("COPIED", "copied"), + ("RECREATED_VIEWS", "recreated_view"), + ("RECREATED_FUNCTIONS", "recreated_function"), +]; + +/// Status footer labels and the status each counts. +pub(crate) const STATUS_SUMMARY: [(&str, CloneStatusKind); 3] = [ + ("PASS", CloneStatusKind::Success), + ("WARN", CloneStatusKind::Warning), + ("FAIL", CloneStatusKind::Failed), +]; diff --git a/crates/sqlbuild-core/src/clone_text/errors.rs b/crates/sqlbuild-core/src/clone_text/errors.rs new file mode 100644 index 0000000000..03c08d6809 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/errors.rs @@ -0,0 +1,8 @@ +//! Clone request validation errors. + +/// A clone request the targets do not allow, with its diagnostic code. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CloneTargetError { + pub code: &'static str, + pub message: String, +} diff --git a/crates/sqlbuild-core/src/clone_text/main/clone_exit_code.rs b/crates/sqlbuild-core/src/clone_text/main/clone_exit_code.rs new file mode 100644 index 0000000000..c4b017d461 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/clone_exit_code.rs @@ -0,0 +1,13 @@ +//! The exit code of `sqb clone`. + +use crate::clone_text::models::{CloneItemText, CloneStatusKind}; + +/// 1 when any item failed, else 0. +#[must_use] +pub fn clone_exit_code(items: &[CloneItemText]) -> i32 { + i32::from( + items + .iter() + .any(|item| item.status == CloneStatusKind::Failed), + ) +} diff --git a/crates/sqlbuild-core/src/clone_text/main/clone_relation_flow_text.rs b/crates/sqlbuild-core/src/clone_text/main/clone_relation_flow_text.rs new file mode 100644 index 0000000000..b2ae1c8b00 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/clone_relation_flow_text.rs @@ -0,0 +1,19 @@ +//! The plain origin-to-destination text of one clone item. + +use crate::clone_text::_helpers::quoted::clone_relation_display_name; + +/// `origin -> destination`, or the lowercased name when either relation is unknown. +#[must_use] +pub fn clone_relation_flow_text( + name: &str, + (origin, destination): (Option<&str>, Option<&str>), +) -> String { + match (origin, destination) { + (Some(origin), Some(destination)) => format!( + "{} -> {}", + clone_relation_display_name(origin), + clone_relation_display_name(destination) + ), + _ => name.to_lowercase(), + } +} diff --git a/crates/sqlbuild-core/src/clone_text/main/mod.rs b/crates/sqlbuild-core/src/clone_text/main/mod.rs new file mode 100644 index 0000000000..6f6ec74604 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/mod.rs @@ -0,0 +1,9 @@ +pub mod clone_exit_code; +pub mod clone_relation_flow_text; +pub mod render_clone_completion; +pub mod render_clone_fingerprint_interrupted_line; +pub mod render_clone_fingerprint_progress_line; +pub mod render_clone_header; +pub mod render_clone_item_line; +pub mod validate_clone_policy; +pub mod validate_clone_target_names; diff --git a/crates/sqlbuild-core/src/clone_text/main/render_clone_completion.rs b/crates/sqlbuild-core/src/clone_text/main/render_clone_completion.rs new file mode 100644 index 0000000000..9d31dc66ca --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/render_clone_completion.rs @@ -0,0 +1,58 @@ +//! The final clone output: non-success messages, then the completion line with counts. + +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, CompletionState, SummaryCount}; +use crate::clone_text::constants::{ACTION_SUMMARY, STATUS_SUMMARY}; +use crate::clone_text::models::{CloneItemText, CloneStatusKind}; + +fn count(items: &[CloneItemText], predicate: impl Fn(&CloneItemText) -> bool) -> i64 { + i64::try_from(items.iter().filter(|item| predicate(item)).count()).unwrap_or(i64::MAX) +} + +/// Messages of items that did not succeed, a blank line, and the completion summary. +#[must_use] +pub fn render_clone_completion( + items: &[CloneItemText], + elapsed_seconds: Option, + style: CliStyle, +) -> String { + let mut text: String = String::new(); + for item in items { + if let (Some(message), false) = (&item.message, item.status == CloneStatusKind::Success) { + text.push_str(&format!( + " {} {}\n", + style.object_name(&item.name), + style.muted(message) + )); + } + } + let mut counts: Vec = ACTION_SUMMARY + .iter() + .map(|(label, action)| SummaryCount { + label: (*label).to_owned(), + value: count(items, |item| item.action == *action), + }) + .collect(); + counts.extend(STATUS_SUMMARY.iter().map(|(label, status)| SummaryCount { + label: (*label).to_owned(), + value: count(items, |item| item.status == *status), + })); + counts.push(SummaryCount { + label: "TOTAL".to_owned(), + value: count(items, |_| true), + }); + let elapsed: Option = elapsed_seconds.map(|seconds| format!("({seconds:.2}s)")); + let footer: String = format_summary_footer(style, &counts, elapsed.as_deref()); + let warnings: i64 = count(items, |item| item.status == CloneStatusKind::Warning); + let failures: i64 = count(items, |item| item.status == CloneStatusKind::Failed); + let (state, label) = match (warnings, failures) { + (0, 0) => (CompletionState::Ok, "Completed successfully"), + (_, 0) => (CompletionState::Warn, "Completed with warnings"), + _ => (CompletionState::Fail, "Completed with errors"), + }; + text.push('\n'); + text.push_str(&format_completion_line(style, state, label, Some(&footer))); + text.push('\n'); + text +} diff --git a/crates/sqlbuild-core/src/clone_text/main/render_clone_fingerprint_interrupted_line.rs b/crates/sqlbuild-core/src/clone_text/main/render_clone_fingerprint_interrupted_line.rs new file mode 100644 index 0000000000..5b5a2a0672 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/render_clone_fingerprint_interrupted_line.rs @@ -0,0 +1,33 @@ +//! Bounded context for interrupted clone fingerprint propagation. + +use crate::cli_text::models::CliStyle; +use crate::clone_text::constants::{ + FINGERPRINT_LABEL, FINGERPRINT_LABEL_WIDTH, PENDING_IDENTITY_LIMIT, +}; +use crate::clone_text::models::FingerprintProgress; + +/// The confirmed count and up to ten pending identities; none once everything was confirmed. +#[must_use] +pub fn render_clone_fingerprint_interrupted_line( + progress: &FingerprintProgress, + style: CliStyle, +) -> Option { + if progress.total <= progress.completed { + return None; + } + let shown: usize = progress + .pending_identities + .len() + .min(PENDING_IDENTITY_LIMIT); + let remaining: usize = progress.pending_identities.len() - shown; + let mut pending: String = progress.pending_identities[..shown].join(", "); + if remaining > 0 { + pending = format!("{pending} (+{remaining} more)"); + } + Some(format!( + " {FINGERPRINT_LABEL: String { + let width: usize = total.to_string().len(); + format!( + " {FINGERPRINT_LABEL:width$}/{total}] propagated {}", + style.status("OK") + ) +} diff --git a/crates/sqlbuild-core/src/clone_text/main/render_clone_header.rs b/crates/sqlbuild-core/src/clone_text/main/render_clone_header.rs new file mode 100644 index 0000000000..9ec2a9a210 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/render_clone_header.rs @@ -0,0 +1,22 @@ +//! The header naming the clone origin, destination and resource count. + +use crate::cli_text::models::CliStyle; + +/// `sqb clone origin= destination= ( resources)`. +#[must_use] +pub fn render_clone_header( + (origin, destination): (&str, &str), + total: usize, + style: CliStyle, +) -> String { + let plural: &str = if total == 1 { "" } else { "s" }; + format!( + "{} {}{} {}{} {}", + style.title("sqb clone"), + style.label("origin="), + style.value(origin), + style.label("destination="), + style.value(destination), + style.muted(&format!("({total} resource{plural})")) + ) +} diff --git a/crates/sqlbuild-core/src/clone_text/main/render_clone_item_line.rs b/crates/sqlbuild-core/src/clone_text/main/render_clone_item_line.rs new file mode 100644 index 0000000000..237e31d9cc --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/render_clone_item_line.rs @@ -0,0 +1,50 @@ +//! One streamed clone progress line. + +use crate::cli_text::models::CliStyle; +use crate::clone_text::_helpers::quoted::clone_relation_display_name; +use crate::clone_text::constants::ACTION_WIDTH; +use crate::clone_text::main::clone_relation_flow_text::clone_relation_flow_text; +use crate::clone_text::models::{CloneItemText, CloneStatusKind}; + +/// Position, action, origin to destination padded to `relation_width`, status and duration. +#[must_use] +pub fn render_clone_item_line( + (index, total): (usize, usize), + item: &CloneItemText, + relation_width: usize, + style: CliStyle, +) -> String { + let position: String = format!("{index:>width$}/{total}", width = total.to_string().len()); + let relations = ( + item.origin_relation.as_deref(), + item.destination_relation.as_deref(), + ); + let plain_flow: String = clone_relation_flow_text(&item.name, relations); + let flow: String = match relations { + (Some(origin), Some(destination)) => format!( + "{} {} {}", + style.object_name(&clone_relation_display_name(origin)), + style.muted("->"), + style.object_name(&clone_relation_display_name(destination)) + ), + _ => style.object_name(&plain_flow), + }; + let plain_width: usize = plain_flow.chars().count(); + let relation_padding: String = " ".repeat(relation_width.max(plain_width) - plain_width); + let action_padding: String = + " ".repeat(ACTION_WIDTH.saturating_sub(item.action.chars().count())); + let status: &str = match item.status { + CloneStatusKind::Warning => "WARN", + CloneStatusKind::Failed => "FAIL", + CloneStatusKind::Success => "OK", + }; + let duration: String = item.duration_seconds.map_or_else(String::new, |seconds| { + format!(" {}", style.muted(&format!("{seconds:.2}s"))) + }); + format!( + " {} {}{action_padding} {flow}{relation_padding} {}{duration}", + style.muted(&position), + style.accent(&item.action), + style.status(status) + ) +} diff --git a/crates/sqlbuild-core/src/clone_text/main/validate_clone_policy.rs b/crates/sqlbuild-core/src/clone_text/main/validate_clone_policy.rs new file mode 100644 index 0000000000..bdd4e2abf0 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/validate_clone_policy.rs @@ -0,0 +1,29 @@ +//! Check that the targets' clone policies allow the requested direction. + +use crate::clone_text::errors::CloneTargetError; + +/// C404 when the origin may not be cloned from, C405 when the destination may not be cloned to. +pub fn validate_clone_policy( + (origin, destination): (&str, &str), + (origin_allowed, destination_allowed): (bool, bool), +) -> Result<(), CloneTargetError> { + if !origin_allowed { + return Err(CloneTargetError { + code: "C404", + message: format!( + "target '{origin}' is not allowed as a clone origin target; \ + set targets.{origin}.clone.allow_as_clone_origin = true" + ), + }); + } + if !destination_allowed { + return Err(CloneTargetError { + code: "C405", + message: format!( + "target '{destination}' is not allowed as a clone destination target; \ + set targets.{destination}.clone.allow_as_clone_destination = true" + ), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/clone_text/main/validate_clone_target_names.rs b/crates/sqlbuild-core/src/clone_text/main/validate_clone_target_names.rs new file mode 100644 index 0000000000..1f6a2b1bf8 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/main/validate_clone_target_names.rs @@ -0,0 +1,31 @@ +//! Check that clone origin and destination name distinct, known targets. + +use std::collections::BTreeSet; + +use crate::clone_text::errors::CloneTargetError; + +/// C401 for the same target, C402 for an unknown origin, C403 for an unknown destination. +pub fn validate_clone_target_names( + (origin, destination): (&str, &str), + known: &BTreeSet, +) -> Result<(), CloneTargetError> { + if origin == destination { + return Err(CloneTargetError { + code: "C401", + message: "clone requires different --from and --to targets".to_owned(), + }); + } + if !known.contains(origin) { + return Err(CloneTargetError { + code: "C402", + message: format!("unknown target '{origin}'"), + }); + } + if !known.contains(destination) { + return Err(CloneTargetError { + code: "C403", + message: format!("unknown target '{destination}'"), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/clone_text/mod.rs b/crates/sqlbuild-core/src/clone_text/mod.rs new file mode 100644 index 0000000000..8ea04437d8 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/mod.rs @@ -0,0 +1,10 @@ +//! `sqb clone`: target validation, streamed progress lines, the completion summary and exit code. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/clone_text/models.rs b/crates/sqlbuild-core/src/clone_text/models.rs new file mode 100644 index 0000000000..fa27816787 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/models.rs @@ -0,0 +1,42 @@ +//! Clone item results as the command renders them. + +/// The outcome of one cloned resource. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CloneStatusKind { + Success, + Warning, + Failed, +} + +impl CloneStatusKind { + /// The status from its value, treating anything unknown as success. + #[must_use] + pub fn from_value(value: &str) -> Self { + match value { + "warning" => Self::Warning, + "failed" => Self::Failed, + _ => Self::Success, + } + } +} + +/// One clone item result. +#[derive(Clone, Debug, PartialEq)] +pub struct CloneItemText { + pub name: String, + /// The `CloneAction` value, such as `copied`. + pub action: String, + pub status: CloneStatusKind, + pub message: Option, + pub origin_relation: Option, + pub destination_relation: Option, + pub duration_seconds: Option, +} + +/// Fingerprint propagation progress: confirmed count, total, and identities still pending. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FingerprintProgress { + pub completed: i64, + pub total: i64, + pub pending_identities: Vec, +} diff --git a/crates/sqlbuild-core/src/clone_text/tests.rs b/crates/sqlbuild-core/src/clone_text/tests.rs new file mode 100644 index 0000000000..1f95e219da --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_clone_text.rs"] +mod clone_text; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/clone_text/tests/helpers.rs b/crates/sqlbuild-core/src/clone_text/tests/helpers.rs new file mode 100644 index 0000000000..924194644c --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/tests/helpers.rs @@ -0,0 +1,29 @@ +use crate::clone_text::models::{CloneItemText, CloneStatusKind}; + +/// An item with an action and status and no relations or message. +pub(super) fn item(name: &str, action: &str, status: CloneStatusKind) -> CloneItemText { + CloneItemText { + name: name.to_owned(), + action: action.to_owned(), + status, + message: None, + origin_relation: None, + destination_relation: None, + duration_seconds: None, + } +} + +/// An item with a message. +pub(super) fn with_message(item: CloneItemText, message: &str) -> CloneItemText { + CloneItemText { + message: Some(message.to_owned()), + ..item + } +} + +/// Twelve pending model identities. +pub(super) fn pending_identities() -> Vec { + (0..12) + .map(|index| format!("model:orders_{index}")) + .collect() +} diff --git a/crates/sqlbuild-core/src/clone_text/tests/test_clone_text.rs b/crates/sqlbuild-core/src/clone_text/tests/test_clone_text.rs new file mode 100644 index 0000000000..65d91bb78d --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/tests/test_clone_text.rs @@ -0,0 +1,292 @@ +use std::collections::BTreeSet; + +use crate::cli_text::models::CliStyle; +use crate::clone_text::_helpers::quoted::clone_relation_display_name; +use crate::clone_text::main::clone_exit_code::clone_exit_code; +use crate::clone_text::main::render_clone_completion::render_clone_completion; +use crate::clone_text::main::render_clone_fingerprint_interrupted_line::render_clone_fingerprint_interrupted_line; +use crate::clone_text::main::render_clone_fingerprint_progress_line::render_clone_fingerprint_progress_line; +use crate::clone_text::main::render_clone_item_line::render_clone_item_line; +use crate::clone_text::main::validate_clone_policy::validate_clone_policy; +use crate::clone_text::main::validate_clone_target_names::validate_clone_target_names; +use crate::clone_text::models::{CloneItemText, CloneStatusKind, FingerprintProgress}; +use crate::clone_text::tests::helpers::{item, pending_identities, with_message}; +use crate::clone_text::tests::test_types::{ + CompletionTestCase, DisplayNameTestCase, FingerprintTestCase, ItemLineTestCase, TargetTestCase, +}; + +#[test] +fn given_relation_when_displaying_then_lowercases_only_unquoted_parts() { + let test_cases = [ + DisplayNameTestCase { + description: "plain identifiers are lowercased", + relation: "PROD.Fact_Orders", + expected_name: "prod.fact_orders", + }, + DisplayNameTestCase { + description: "double, backtick and bracket quotes keep their case", + relation: "DB.\"Mixed\"\"Case\".`Tick`.[Br]]ack]", + expected_name: "db.\"Mixed\"\"Case\".`Tick`.[Br]]ack]", + }, + DisplayNameTestCase { + description: "an unclosed quote is ordinary text", + relation: "A.\"Open", + expected_name: "a.\"open", + }, + DisplayNameTestCase { + description: "a trailing doubled quote backtracks to the first closing quote", + relation: "\"Ab\"\"C", + expected_name: "\"Ab\"\"c", + }, + ]; + + for test_case in test_cases { + assert_eq!( + clone_relation_display_name(test_case.relation), + test_case.expected_name, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_item_when_rendering_line_then_aligns_action_and_relations() { + let test_cases = [ + ItemLineTestCase { + description: "relations, duration and padding to the widest flow", + position: (2, 12), + item: CloneItemText { + origin_relation: Some("PROD.Orders".to_owned()), + destination_relation: Some("dev.orders".to_owned()), + duration_seconds: Some(0.125), + ..item("orders", "copied", CloneStatusKind::Success) + }, + relation_width: 30, + use_color: false, + expected_line: " 2/12 copied prod.orders -> dev.orders OK 0.12s", + }, + ItemLineTestCase { + description: "quoted identifiers keep their case in the flow", + position: (5, 5), + item: CloneItemText { + origin_relation: Some("PROD.DBO.[My]]Table]".to_owned()), + destination_relation: Some("DEV.MAIN.\"Orders\"".to_owned()), + ..item("orders", "cloned", CloneStatusKind::Success) + }, + relation_width: 0, + use_color: false, + expected_line: " 5/5 cloned prod.dbo.[My]]Table] -> dev.main.\"Orders\" OK", + }, + ItemLineTestCase { + description: "missing relations fall back to the lowercased name", + position: (1, 1), + item: item("Stg", "warning_missing_source", CloneStatusKind::Warning), + relation_width: 0, + use_color: false, + expected_line: " 1/1 warning_missing_source stg WARN", + }, + ItemLineTestCase { + description: "colour styles position, action, arrow and status", + position: (1, 1), + item: CloneItemText { + origin_relation: Some("a.x".to_owned()), + destination_relation: Some("b.x".to_owned()), + ..item("x", "failed", CloneStatusKind::Failed) + }, + relation_width: 0, + use_color: true, + expected_line: " \u{1b}[2m1/1\u{1b}[0m \u{1b}[34mfailed\u{1b}[0m a.x \u{1b}[2m->\u{1b}[0m b.x \u{1b}[38;5;167mFAIL\u{1b}[0m", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_clone_item_line( + test_case.position, + &test_case.item, + test_case.relation_width, + CliStyle { + use_color: test_case.use_color + }, + ), + test_case.expected_line, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_items_when_completing_then_reports_messages_counts_and_exit_code() { + let test_cases = [ + CompletionTestCase { + description: "a failure reports errors and messages of non-success items", + items: vec![ + item("fact_orders", "cloned", CloneStatusKind::Success), + with_message( + item( + "stg_orders", + "warning_missing_source", + CloneStatusKind::Warning, + ), + "missing in origin environment", + ), + with_message( + item("dim_customers", "failed", CloneStatusKind::Failed), + "boom", + ), + with_message( + item("add_tax", "recreated_function", CloneStatusKind::Success), + "ignored", + ), + ], + elapsed_seconds: None, + use_color: false, + expected_text: " stg_orders missing in origin environment\n dim_customers boom\n\n\ + \u{2717} Completed with errors CLONED=1 COPIED=0 RECREATED_VIEWS=0 \ + RECREATED_FUNCTIONS=1 PASS=2 WARN=1 FAIL=1 TOTAL=4\n", + expected_exit_code: 1, + }, + CompletionTestCase { + description: "success shows the doubly parenthesised elapsed time", + items: vec![item("orders", "copied", CloneStatusKind::Success)], + elapsed_seconds: Some(1.5), + use_color: false, + expected_text: "\n\u{2713} Completed successfully CLONED=0 COPIED=1 RECREATED_VIEWS=0 \ + RECREATED_FUNCTIONS=0 PASS=1 WARN=0 FAIL=0 TOTAL=1 ((1.50s))\n", + expected_exit_code: 0, + }, + CompletionTestCase { + description: "colour styles the failure glyph and label", + items: vec![item("orders", "failed", CloneStatusKind::Failed)], + elapsed_seconds: None, + use_color: true, + expected_text: "\n\u{1b}[38;5;167m\u{2717}\u{1b}[0m \u{1b}[38;5;167m\u{1b}[1mCompleted with errors\u{1b}[0m \ + \u{1b}[2mCLONED=\u{1b}[0m\u{1b}[34m0\u{1b}[0m \u{1b}[2mCOPIED=\u{1b}[0m\u{1b}[34m0\u{1b}[0m \ + \u{1b}[2mRECREATED_VIEWS=\u{1b}[0m\u{1b}[34m0\u{1b}[0m \ + \u{1b}[2mRECREATED_FUNCTIONS=\u{1b}[0m\u{1b}[34m0\u{1b}[0m \ + \u{1b}[2mPASS=\u{1b}[0m\u{1b}[32m0\u{1b}[0m \u{1b}[2mWARN=\u{1b}[0m\u{1b}[33m0\u{1b}[0m \ + \u{1b}[2mFAIL=\u{1b}[0m\u{1b}[38;5;167m1\u{1b}[0m \u{1b}[2mTOTAL=\u{1b}[0m\u{1b}[34m1\u{1b}[0m\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + render_clone_completion( + &test_case.items, + test_case.elapsed_seconds, + CliStyle { + use_color: test_case.use_color + }, + ), + clone_exit_code(&test_case.items), + ), + ( + test_case.expected_text.to_owned(), + test_case.expected_exit_code + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_fingerprint_progress_when_rendering_then_reports_confirmed_and_pending() { + let test_cases = [ + FingerprintTestCase { + description: "progress pads the count and interruption bounds pending identities", + progress: FingerprintProgress { + completed: 50, + total: 386, + pending_identities: pending_identities(), + }, + expected_progress_line: " fingerprint[ 50/386] propagated OK", + expected_interrupted_line: Some( + " fingerprint[50/386] FAIL not confirmed: model:orders_0, \ + model:orders_1, model:orders_2, model:orders_3, model:orders_4, model:orders_5, \ + model:orders_6, model:orders_7, model:orders_8, model:orders_9 (+2 more)", + ), + }, + FingerprintTestCase { + description: "a fully confirmed propagation reports no interruption", + progress: FingerprintProgress { + completed: 3, + total: 3, + pending_identities: Vec::new(), + }, + expected_progress_line: " fingerprint[3/3] propagated OK", + expected_interrupted_line: None, + }, + ]; + + for test_case in test_cases { + let style: CliStyle = CliStyle { use_color: false }; + assert_eq!( + ( + render_clone_fingerprint_progress_line( + test_case.progress.completed, + test_case.progress.total, + style, + ), + render_clone_fingerprint_interrupted_line(&test_case.progress, style), + ), + ( + test_case.expected_progress_line.to_owned(), + test_case.expected_interrupted_line.map(str::to_owned), + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_targets_when_validating_then_reports_the_first_problem() { + let known: BTreeSet = ["dev", "prod"].into_iter().map(str::to_owned).collect(); + let test_cases = [ + TargetTestCase { + description: "known distinct targets with allowed policies pass", + targets: ("prod", "dev"), + policy: (true, true), + expected_codes: (None, None), + }, + TargetTestCase { + description: "the same target is rejected first", + targets: ("dev", "dev"), + policy: (true, true), + expected_codes: (Some("C401"), None), + }, + TargetTestCase { + description: "unknown origin", + targets: ("nope", "dev"), + policy: (false, true), + expected_codes: (Some("C402"), Some("C404")), + }, + TargetTestCase { + description: "unknown destination and denied destination policy", + targets: ("prod", "nope"), + policy: (true, false), + expected_codes: (Some("C403"), Some("C405")), + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + validate_clone_target_names(test_case.targets, &known) + .err() + .map(|error| error.code), + validate_clone_policy(test_case.targets, test_case.policy) + .err() + .map(|error| error.code), + ), + test_case.expected_codes, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/clone_text/tests/test_types.rs b/crates/sqlbuild-core/src/clone_text/tests/test_types.rs new file mode 100644 index 0000000000..904b525025 --- /dev/null +++ b/crates/sqlbuild-core/src/clone_text/tests/test_types.rs @@ -0,0 +1,39 @@ +use crate::clone_text::models::{CloneItemText, FingerprintProgress}; + +pub(super) struct DisplayNameTestCase { + pub(super) description: &'static str, + pub(super) relation: &'static str, + pub(super) expected_name: &'static str, +} + +pub(super) struct ItemLineTestCase { + pub(super) description: &'static str, + pub(super) position: (usize, usize), + pub(super) item: CloneItemText, + pub(super) relation_width: usize, + pub(super) use_color: bool, + pub(super) expected_line: &'static str, +} + +pub(super) struct CompletionTestCase { + pub(super) description: &'static str, + pub(super) items: Vec, + pub(super) elapsed_seconds: Option, + pub(super) use_color: bool, + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} + +pub(super) struct FingerprintTestCase { + pub(super) description: &'static str, + pub(super) progress: FingerprintProgress, + pub(super) expected_progress_line: &'static str, + pub(super) expected_interrupted_line: Option<&'static str>, +} + +pub(super) struct TargetTestCase { + pub(super) description: &'static str, + pub(super) targets: (&'static str, &'static str), + pub(super) policy: (bool, bool), + pub(super) expected_codes: (Option<&'static str>, Option<&'static str>), +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 5afcac9c29..6b08424e07 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -2,7 +2,9 @@ #![forbid(unsafe_code)] +pub mod check_command; pub mod cli_text; +pub mod clone_text; pub mod constants; pub mod cost_report; pub mod debug_report; @@ -10,11 +12,15 @@ pub mod freshness_text; pub mod integration_results; pub mod janitor_text; pub mod json; +pub mod load_command; pub mod models; pub mod panics; pub mod posix_path; pub mod progress; +pub mod redaction; +pub mod seed_command; pub mod terminal_events; +pub mod test_command; pub mod text; #[cfg(test)] diff --git a/crates/sqlbuild-core/src/load_command/_helpers/graph.rs b/crates/sqlbuild-core/src/load_command/_helpers/graph.rs new file mode 100644 index 0000000000..2d337e6e96 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/_helpers/graph.rs @@ -0,0 +1,116 @@ +//! Loader graph lookups: dependencies, closures and execution order. + +use std::collections::{BTreeSet, HashMap}; + +use crate::load_command::models::LoadGraph; + +/// Indexed views of a load graph, as the Python dictionaries keyed them. +pub(crate) struct GraphIndex { + /// Managed source name to its loader; a later duplicate wins. + pub(crate) source_loader: HashMap, + /// Managed source names in declaration order, first occurrence only. + pub(crate) source_order: Vec, + /// Loader name to its dependencies; a later duplicate wins. + pub(crate) upstream: HashMap>, + /// Loader name to its sources in declaration order. + pub(crate) sources_by_loader: HashMap>, +} + +impl GraphIndex { + pub(crate) fn new(graph: &LoadGraph) -> Self { + let mut source_loader: HashMap = HashMap::new(); + let mut source_order: Vec = Vec::new(); + for (name, loader) in &graph.sources { + if source_loader.insert(name.clone(), loader.clone()).is_none() { + source_order.push(name.clone()); + } + } + let mut sources_by_loader: HashMap> = HashMap::new(); + for name in &source_order { + sources_by_loader + .entry(source_loader[name].clone()) + .or_default() + .push(name.clone()); + } + Self { + source_loader, + source_order, + upstream: graph.loaders.iter().cloned().collect(), + sources_by_loader, + } + } + + pub(crate) fn is_loader(&self, name: &str) -> bool { + self.upstream.contains_key(name) + } + + pub(crate) fn dependencies(&self, loader: &str) -> &[String] { + self.upstream.get(loader).map_or(&[], Vec::as_slice) + } + + /// The loader and every loader it transitively depends on. + pub(crate) fn upstream_closure(&self, loader: &str) -> BTreeSet { + let mut closure: BTreeSet = BTreeSet::new(); + let mut pending: Vec = vec![loader.to_owned()]; + while let Some(next) = pending.pop() { + if closure.insert(next.clone()) { + pending.extend(self.dependencies(&next).iter().cloned()); + } + } + closure + } + + /// The loader, every loader depending on it transitively, and all their sources. + pub(crate) fn downstream_closure(&self, loader: &str) -> (BTreeSet, BTreeSet) { + let mut loaders: BTreeSet = BTreeSet::new(); + let mut pending: Vec = vec![loader.to_owned()]; + while let Some(next) = pending.pop() { + if !loaders.insert(next.clone()) { + continue; + } + for (candidate, dependencies) in &self.upstream { + if dependencies.contains(&next) { + pending.push(candidate.clone()); + } + } + } + let sources: BTreeSet = loaders + .iter() + .flat_map(|name| { + self.sources_by_loader + .get(name) + .into_iter() + .flatten() + .cloned() + }) + .collect(); + (loaders, sources) + } + + /// `names` ordered dependencies first, visiting in sorted order. + pub(crate) fn topological_order(&self, names: &BTreeSet) -> Vec { + let mut ordered: Vec = Vec::new(); + let mut visited: BTreeSet = BTreeSet::new(); + for name in names { + self.visit(name, names, (&mut ordered, &mut visited)); + } + ordered + } + + fn visit( + &self, + name: &str, + names: &BTreeSet, + (ordered, visited): (&mut Vec, &mut BTreeSet), + ) { + if !visited.insert(name.to_owned()) { + return; + } + for dependency in self.dependencies(name) { + if names.contains(dependency) { + self.visit(dependency, names, (ordered, visited)); + } + } + ordered.push(name.to_owned()); + } +} diff --git a/crates/sqlbuild-core/src/load_command/_helpers/mod.rs b/crates/sqlbuild-core/src/load_command/_helpers/mod.rs new file mode 100644 index 0000000000..06011aeaf4 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod graph; +pub(crate) mod padding; +pub(crate) mod selection_sets; diff --git a/crates/sqlbuild-core/src/load_command/_helpers/padding.rs b/crates/sqlbuild-core/src/load_command/_helpers/padding.rs new file mode 100644 index 0000000000..2af6b2a40f --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/_helpers/padding.rs @@ -0,0 +1,24 @@ +//! Python format-spec padding and digit grouping. + +/// `f"{text: String { + let length: usize = text.chars().count(); + format!("{text}{}", " ".repeat(width.saturating_sub(length))) +} + +/// `f"{value:,}"`: the integer with comma-separated thousands. +pub(crate) fn group_thousands(value: i64) -> String { + let digits: String = value.unsigned_abs().to_string(); + let mut grouped: String = String::new(); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + grouped.push(','); + } + grouped.push(digit); + } + if value < 0 { + format!("-{grouped}") + } else { + grouped + } +} diff --git a/crates/sqlbuild-core/src/load_command/_helpers/selection_sets.rs b/crates/sqlbuild-core/src/load_command/_helpers/selection_sets.rs new file mode 100644 index 0000000000..17635232b1 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/_helpers/selection_sets.rs @@ -0,0 +1,190 @@ +//! Applying load selectors and excludes to the selected source and loader sets. + +use std::collections::BTreeSet; + +use crate::load_command::_helpers::graph::GraphIndex; +use crate::load_command::constants::UPSTREAM_MARK; +use crate::load_command::errors::UnknownLoadSelector; + +/// Selected sources and loaders, and the loaders selected by name. +#[derive(Default)] +pub(crate) struct Selection { + pub(crate) sources: BTreeSet, + pub(crate) loaders: BTreeSet, + pub(crate) direct_loaders: BTreeSet, + pub(crate) excluded_loaders: BTreeSet, +} + +/// The selector name and whether it asks for upstream and downstream nodes. +fn parse(selector: &str) -> (String, bool, bool) { + let stripped: &str = selector.trim(); + ( + stripped.trim_matches(UPSTREAM_MARK).to_owned(), + stripped.starts_with(UPSTREAM_MARK), + stripped.ends_with(UPSTREAM_MARK), + ) +} + +fn validate(index: &GraphIndex, name: &str) -> Result<(), UnknownLoadSelector> { + if index.source_loader.contains_key(name) || index.is_loader(name) { + return Ok(()); + } + Err(UnknownLoadSelector { + name: name.to_owned(), + }) +} + +/// The loader a downstream selector starts from. +fn downstream_root<'a>(index: &'a GraphIndex, name: &'a str) -> &'a str { + if index.is_loader(name) { + return name; + } + &index.source_loader[name] +} + +impl Selection { + fn add_loader(&mut self, index: &GraphIndex, (loader, upstream): (&str, bool)) { + self.loaders.insert(loader.to_owned()); + if upstream { + self.loaders.extend(index.upstream_closure(loader)); + } + } + + fn select_name(&mut self, index: &GraphIndex, (name, upstream): (&str, bool)) { + if let Some(loader) = index.source_loader.get(name) { + self.sources.insert(name.to_owned()); + self.add_loader(index, (loader, upstream)); + } else if let Some(sources) = index.sources_by_loader.get(name) { + self.sources.extend(sources.iter().cloned()); + self.add_loader(index, (name, upstream)); + } else if index.is_loader(name) { + self.direct_loaders.insert(name.to_owned()); + self.add_loader(index, (name, upstream)); + } + } + + /// Apply each selector in order. + pub(crate) fn select( + &mut self, + index: &GraphIndex, + selectors: &[String], + ) -> Result<(), UnknownLoadSelector> { + for selector in selectors { + let (name, upstream, downstream) = parse(selector); + validate(index, &name)?; + self.select_name(index, (&name, upstream)); + if downstream { + let (loaders, sources) = index.downstream_closure(downstream_root(index, &name)); + self.loaders.extend(loaders); + self.sources.extend(sources); + } + } + Ok(()) + } + + fn exclude_name(&mut self, index: &GraphIndex, name: &str) { + if let Some(loader) = index.source_loader.get(name) { + self.sources.remove(name); + let still_used: bool = index + .sources_by_loader + .get(loader) + .into_iter() + .flatten() + .any(|source| self.sources.contains(source)); + if !still_used { + self.loaders.remove(loader); + } + } else if let Some(sources) = index.sources_by_loader.get(name) { + for source in sources { + self.sources.remove(source); + } + self.loaders.remove(name); + self.excluded_loaders.insert(name.to_owned()); + } else if index.is_loader(name) { + self.loaders.remove(name); + self.excluded_loaders.insert(name.to_owned()); + } + } + + /// Apply each exclude in order. + pub(crate) fn exclude( + &mut self, + index: &GraphIndex, + excludes: &[String], + ) -> Result<(), UnknownLoadSelector> { + for selector in excludes { + let (name, _, downstream) = parse(selector); + validate(index, &name)?; + self.exclude_name(index, &name); + if downstream { + let (loaders, sources) = index.downstream_closure(downstream_root(index, &name)); + self.loaders.retain(|loader| !loaders.contains(loader)); + self.sources.retain(|source| !sources.contains(source)); + } + } + Ok(()) + } + + /// Whether a dependency of `loader` is unselected and blocked. + fn has_blocked_dependency( + &self, + index: &GraphIndex, + (loader, blocked): (&str, &BTreeSet), + ) -> bool { + for dependency in index.dependencies(loader) { + if !self.loaders.contains(dependency) && blocked.contains(dependency) { + return true; + } + } + false + } + + /// Drop loaders whose dependency was excluded or dropped, and the sources they run. + pub(crate) fn prune_missing_dependencies(&mut self, index: &GraphIndex) { + let mut blocked: BTreeSet = self.excluded_loaders.clone(); + loop { + let missing: Vec = self + .loaders + .iter() + .filter(|loader| self.has_blocked_dependency(index, (loader, &blocked))) + .cloned() + .collect(); + let orphaned: Vec = self + .sources + .iter() + .filter(|source| !self.loaders.contains(&index.source_loader[*source])) + .cloned() + .collect(); + if missing.is_empty() && orphaned.is_empty() { + return; + } + for loader in missing { + self.loaders.remove(&loader); + blocked.insert(loader); + } + for source in orphaned { + self.sources.remove(&source); + } + } + } + + /// Keep only loaders a selected source or a by-name loader needs. + pub(crate) fn prune_unneeded_loaders(&mut self, index: &GraphIndex) { + let mut needed: BTreeSet = self.direct_loaders.clone(); + for source in &self.sources { + let loader: &String = &index.source_loader[source]; + if self.loaders.contains(loader) { + needed.insert(loader.clone()); + } + } + let mut pending: Vec = needed.iter().cloned().collect(); + while let Some(loader) = pending.pop() { + for dependency in index.dependencies(&loader) { + if self.loaders.contains(dependency) && needed.insert(dependency.clone()) { + pending.push(dependency.clone()); + } + } + } + self.loaders.retain(|loader| needed.contains(loader)); + } +} diff --git a/crates/sqlbuild-core/src/load_command/constants.rs b/crates/sqlbuild-core/src/load_command/constants.rs new file mode 100644 index 0000000000..2641036745 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/constants.rs @@ -0,0 +1,15 @@ +//! Load output labels and widths. + +pub(crate) const KIND_WIDTH: usize = 10; +pub(crate) const NAME_WIDTH: usize = 30; +pub(crate) const STATUS_WIDTH: usize = 6; +pub(crate) const SPINNER_NAME_WIDTH: usize = 48; +pub(crate) const LOG_LABEL: &str = " log "; +pub(crate) const LOG_CONTINUATION: &str = " "; +pub(crate) const UPSTREAM_MARK: char = '+'; +pub(crate) const SPINNER_FRAMES: [&str; 10] = [ + "\u{280b}", "\u{2819}", "\u{2839}", "\u{2838}", "\u{283c}", "\u{2834}", "\u{2826}", "\u{2827}", + "\u{2807}", "\u{280f}", +]; +pub(crate) const TREE_BRANCH_GLYPH: &str = "\u{251c}\u{2500}\u{2500}"; +pub(crate) const TREE_LAST_GLYPH: &str = "\u{2514}\u{2500}\u{2500}"; diff --git a/crates/sqlbuild-core/src/load_command/errors.rs b/crates/sqlbuild-core/src/load_command/errors.rs new file mode 100644 index 0000000000..ba14a13df2 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/errors.rs @@ -0,0 +1,7 @@ +//! Load selection errors. + +/// A `--select` or `--exclude` name that matches no managed source or loader. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UnknownLoadSelector { + pub name: String, +} diff --git a/crates/sqlbuild-core/src/load_command/main/load_counts.rs b/crates/sqlbuild-core/src/load_command/main/load_counts.rs new file mode 100644 index 0000000000..ec21daf7a5 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/load_counts.rs @@ -0,0 +1,24 @@ +//! Result counts of one load run. + +use crate::load_command::models::{LoadCounts, LoadResultText, LoadStatus}; + +fn count(results: &[LoadResultText], predicate: impl Fn(&LoadResultText) -> bool) -> i64 { + i64::try_from(results.iter().filter(|result| predicate(result)).count()).unwrap_or(i64::MAX) +} + +/// Successes without warnings pass; successes with warnings warn; failures and skips count apart. +#[must_use] +pub fn load_counts(results: &[LoadResultText]) -> LoadCounts { + let success = |result: &LoadResultText| result.status == LoadStatus::Success; + LoadCounts { + pass: count(results, |result| { + success(result) && result.warning_messages.is_empty() + }), + warn: count(results, |result| { + success(result) && !result.warning_messages.is_empty() + }), + fail: count(results, |result| result.status == LoadStatus::Failed), + skip: count(results, |result| result.status == LoadStatus::Skipped), + total: count(results, |_| true), + } +} diff --git a/crates/sqlbuild-core/src/load_command/main/load_exit_code.rs b/crates/sqlbuild-core/src/load_command/main/load_exit_code.rs new file mode 100644 index 0000000000..d3d137a59f --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/load_exit_code.rs @@ -0,0 +1,9 @@ +//! The exit code of `sqb load`. + +use crate::load_command::models::LoadCounts; + +/// 1 when any load failed, else 0. +#[must_use] +pub fn load_exit_code(counts: LoadCounts) -> i32 { + i32::from(counts.fail != 0) +} diff --git a/crates/sqlbuild-core/src/load_command/main/load_reference_loaders.rs b/crates/sqlbuild-core/src/load_command/main/load_reference_loaders.rs new file mode 100644 index 0000000000..36d346555e --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/load_reference_loaders.rs @@ -0,0 +1,28 @@ +//! Unselected upstream loaders that selected loads only reference. + +use std::collections::BTreeSet; + +use crate::load_command::_helpers::graph::GraphIndex; +use crate::load_command::models::LoadGraph; + +/// Loaders upstream of the selected entries' loaders, in dependency order, minus selected names. +#[must_use] +pub fn load_reference_loaders( + graph: &LoadGraph, + selected: &[(String, Option)], +) -> Vec { + let index: GraphIndex = GraphIndex::new(graph); + let mut references: BTreeSet = BTreeSet::new(); + for (_, loader) in selected { + let Some(loader) = loader else { continue }; + for dependency in index.dependencies(loader) { + references.extend(index.upstream_closure(dependency)); + } + } + let selected_names: BTreeSet<&String> = selected.iter().map(|(name, _)| name).collect(); + index + .topological_order(&references) + .into_iter() + .filter(|loader| index.is_loader(loader) && !selected_names.contains(loader)) + .collect() +} diff --git a/crates/sqlbuild-core/src/load_command/main/mod.rs b/crates/sqlbuild-core/src/load_command/main/mod.rs new file mode 100644 index 0000000000..0eb59a050d --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/mod.rs @@ -0,0 +1,10 @@ +pub mod load_counts; +pub mod load_exit_code; +pub mod load_reference_loaders; +pub mod render_empty_load; +pub mod render_load_footer; +pub mod render_load_plan; +pub mod render_load_ready; +pub mod render_load_result; +pub mod render_load_spinner_line; +pub mod select_load_entries; diff --git a/crates/sqlbuild-core/src/load_command/main/render_empty_load.rs b/crates/sqlbuild-core/src/load_command/main/render_empty_load.rs new file mode 100644 index 0000000000..a28b2983b4 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_empty_load.rs @@ -0,0 +1,15 @@ +//! The output of a load with nothing selected. + +use crate::cli_text::models::CliStyle; +use crate::load_command::main::render_load_footer::render_load_footer; +use crate::load_command::models::LoadCounts; + +/// A note that no managed sources were selected, then a zero-count completion line. +#[must_use] +pub fn render_empty_load(style: CliStyle) -> String { + format!( + "{}\n{}", + style.muted(" No managed sources selected."), + render_load_footer(&[], (LoadCounts::default(), 0.0), style) + ) +} diff --git a/crates/sqlbuild-core/src/load_command/main/render_load_footer.rs b/crates/sqlbuild-core/src/load_command/main/render_load_footer.rs new file mode 100644 index 0000000000..482373d225 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_load_footer.rs @@ -0,0 +1,60 @@ +//! The final load summary: loader warnings, then the completion line. + +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, CompletionState, SummaryCount}; +use crate::load_command::models::{LoadCounts, LoadResultText}; + +fn summary_count(label: &str, value: i64) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value, + } +} + +/// Warnings by source when any, a blank line, and the completion line with counts and elapsed time. +#[must_use] +pub fn render_load_footer( + results: &[LoadResultText], + (counts, elapsed_seconds): (LoadCounts, f64), + style: CliStyle, +) -> String { + let mut text: String = String::new(); + for result in results + .iter() + .filter(|result| !result.warning_messages.is_empty()) + { + if text.is_empty() { + text.push_str(&format!("\n{}\n\n", style.warning_strong("Warnings:"))); + } + text.push_str(&format!( + " {} (loader {})\n", + result.source_name, result.loader_name + )); + for message in &result.warning_messages { + text.push_str(&format!(" {message}\n")); + } + } + let footer: String = format_summary_footer( + style, + &[ + summary_count("PASS", counts.pass), + summary_count("WARN", counts.warn), + summary_count("FAIL", counts.fail), + summary_count("SKIP", counts.skip), + summary_count("TOTAL", counts.total), + ], + Some(&format!("{elapsed_seconds:.2}s")), + ); + let (state, label) = if counts.fail != 0 { + (CompletionState::Fail, "Completed with errors") + } else if counts.warn != 0 { + (CompletionState::Warn, "Completed with warnings") + } else { + (CompletionState::Ok, "Completed successfully") + }; + format!( + "{text}\n{}\n", + format_completion_line(style, state, label, Some(&footer)) + ) +} diff --git a/crates/sqlbuild-core/src/load_command/main/render_load_plan.rs b/crates/sqlbuild-core/src/load_command/main/render_load_plan.rs new file mode 100644 index 0000000000..9cc14e8f16 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_load_plan.rs @@ -0,0 +1,41 @@ +//! The selected loader and source names before execution. + +use crate::cli_text::models::CliStyle; +use crate::load_command::constants::{TREE_BRANCH_GLYPH, TREE_LAST_GLYPH}; + +fn names_section(title: &str, names: &[&str], style: CliStyle) -> String { + let mut text: String = format!("{}\n", style.section(&format!("{title} ({})", names.len()))); + for (index, name) in names.iter().enumerate() { + text.push_str(&format!( + "{} {name}\n", + style.muted(if index + 1 == names.len() { + TREE_LAST_GLYPH + } else { + TREE_BRANCH_GLYPH + }) + )); + } + text.push('\n'); + text +} + +/// `Loaders (n)` when any intermediate loaders run, then `Sources (n)`, each as a tree. +#[must_use] +pub fn render_load_plan(entries: &[(String, bool)], style: CliStyle) -> String { + let loaders: Vec<&str> = entries + .iter() + .filter(|(_, loader)| *loader) + .map(|(name, _)| name.as_str()) + .collect(); + let sources: Vec<&str> = entries + .iter() + .filter(|(_, loader)| !*loader) + .map(|(name, _)| name.as_str()) + .collect(); + let mut text: String = String::new(); + if !loaders.is_empty() { + text.push_str(&names_section("Loaders", &loaders, style)); + } + text.push_str(&names_section("Sources", &sources, style)); + text +} diff --git a/crates/sqlbuild-core/src/load_command/main/render_load_ready.rs b/crates/sqlbuild-core/src/load_command/main/render_load_ready.rs new file mode 100644 index 0000000000..da6c6b5875 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_load_ready.rs @@ -0,0 +1,13 @@ +//! The header announcing how many loads were selected. + +use crate::cli_text::main::format_surface_header::format_surface_header; +use crate::cli_text::models::CliStyle; + +/// A blank line, `Load ready selected`, and a blank line. +#[must_use] +pub fn render_load_ready(selected: usize, style: CliStyle) -> String { + format!( + "\n{}\n\n", + format_surface_header(style, "Load ready", &format!("{selected} selected")) + ) +} diff --git a/crates/sqlbuild-core/src/load_command/main/render_load_result.rs b/crates/sqlbuild-core/src/load_command/main/render_load_result.rs new file mode 100644 index 0000000000..9d6110524c --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_load_result.rs @@ -0,0 +1,53 @@ +//! The persistent line of one finished load, with its log blocks and error. + +use crate::cli_text::models::CliStyle; +use crate::load_command::_helpers::padding::{group_thousands, pad_right}; +use crate::load_command::constants::{ + KIND_WIDTH, LOG_CONTINUATION, LOG_LABEL, NAME_WIDTH, STATUS_WIDTH, +}; +use crate::load_command::models::{LoadResultText, LoadStatus}; +use crate::text::main::python_splitlines::python_splitlines; + +fn log_block(message: &str, style: CliStyle) -> String { + let lines: Vec<&str> = python_splitlines(message); + let first: &str = lines.first().copied().unwrap_or(""); + let mut text: String = format!("\n{}{}\n", style.label(LOG_LABEL), style.muted(first)); + for line in lines.iter().skip(1) { + text.push_str(&format!( + "{}\n", + style.muted(&format!("{LOG_CONTINUATION}{line}")) + )); + } + text +} + +/// Position, kind, name, status, duration and rows, then log blocks and the error message. +#[must_use] +pub fn render_load_result( + result: &LoadResultText, + (ordinal, total): (i64, i64), + duration_ms: Option, + style: CliStyle, +) -> String { + let status: String = style.status(match result.status { + LoadStatus::Success => "OK", + LoadStatus::Skipped => "SKIP", + LoadStatus::Failed => "FAIL", + }); + let duration: String = + duration_ms.map_or_else(String::new, |ms| format!("{:.2}s", ms / 1000.0)); + let mut text: String = format!( + " {ordinal}/{total} {}{} {} {duration} rows={}\n", + pad_right(&result.resource_kind, KIND_WIDTH), + pad_right(&result.source_name, NAME_WIDTH), + pad_right(&status, STATUS_WIDTH), + group_thousands(result.rows_loaded) + ); + for message in &result.log_messages { + text.push_str(&log_block(message, style)); + } + if let Some(error) = &result.error_message { + text.push_str(&style.error(&format!(" {error}\n"))); + } + text +} diff --git a/crates/sqlbuild-core/src/load_command/main/render_load_spinner_line.rs b/crates/sqlbuild-core/src/load_command/main/render_load_spinner_line.rs new file mode 100644 index 0000000000..dfaa474b1e --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/render_load_spinner_line.rs @@ -0,0 +1,24 @@ +//! The live row of a running load. + +use crate::cli_text::models::CliStyle; +use crate::load_command::_helpers::padding::pad_right; +use crate::load_command::constants::{KIND_WIDTH, SPINNER_FRAMES, SPINNER_NAME_WIDTH}; +use crate::load_command::models::LoadSpinnerRow; + +/// Carriage return, line clear, and the row with spinner frame `frame`. +#[must_use] +pub fn render_load_spinner_line(row: &LoadSpinnerRow, frame: usize, style: CliStyle) -> String { + let name: String = if row.message.is_empty() { + row.name.clone() + } else { + format!("{} {}", row.name, row.message) + }; + format!( + "\r\u{1b}[K {}/{} {}{} {}", + row.ordinal, + row.total, + pad_right(&row.resource_kind, KIND_WIDTH), + pad_right(&name, SPINNER_NAME_WIDTH), + style.status(SPINNER_FRAMES[frame % SPINNER_FRAMES.len()]) + ) +} diff --git a/crates/sqlbuild-core/src/load_command/main/select_load_entries.rs b/crates/sqlbuild-core/src/load_command/main/select_load_entries.rs new file mode 100644 index 0000000000..69b876aa76 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/main/select_load_entries.rs @@ -0,0 +1,51 @@ +//! The loaders and sources `sqb load` runs for `--select` and `--exclude`. + +use std::collections::BTreeSet; + +use crate::load_command::_helpers::graph::GraphIndex; +use crate::load_command::_helpers::selection_sets::Selection; +use crate::load_command::constants::UPSTREAM_MARK; +use crate::load_command::errors::UnknownLoadSelector; +use crate::load_command::models::{LoadEntry, LoadGraph}; + +/// Intermediate loaders in dependency order, then selected sources in declaration order. +pub fn select_load_entries( + graph: &LoadGraph, + (select, exclude): (&[String], &[String]), +) -> Result, UnknownLoadSelector> { + let index: GraphIndex = GraphIndex::new(graph); + let selectors: Vec = if select.is_empty() { + index + .source_order + .iter() + .map(|name| format!("{UPSTREAM_MARK}{name}")) + .collect() + } else { + select.to_vec() + }; + let mut selection: Selection = Selection::default(); + selection.select(&index, &selectors)?; + selection.exclude(&index, exclude)?; + selection.prune_missing_dependencies(&index); + selection.prune_unneeded_loaders(&index); + let terminal: BTreeSet<&String> = selection + .sources + .iter() + .map(|source| &index.source_loader[source]) + .collect(); + let mut entries: Vec = index + .topological_order(&selection.loaders) + .into_iter() + .filter(|loader| !terminal.contains(loader)) + .map(LoadEntry::Loader) + .collect(); + entries.extend( + index + .source_order + .iter() + .filter(|name| selection.sources.contains(*name)) + .cloned() + .map(LoadEntry::Source), + ); + Ok(entries) +} diff --git a/crates/sqlbuild-core/src/load_command/mod.rs b/crates/sqlbuild-core/src/load_command/mod.rs new file mode 100644 index 0000000000..adc138fc1c --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/mod.rs @@ -0,0 +1,10 @@ +//! `sqb load`: source and loader selection, progress and summary text, and the exit code. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/load_command/models.rs b/crates/sqlbuild-core/src/load_command/models.rs new file mode 100644 index 0000000000..a0b89719e1 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/models.rs @@ -0,0 +1,75 @@ +//! Load selection graphs, entries and results. + +/// The managed sources and loader functions selection works over. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LoadGraph { + /// Managed sources in declaration order, with the loader each runs. + pub sources: Vec<(String, String)>, + /// Loader functions in discovery order, with the loaders each depends on. + pub loaders: Vec<(String, Vec)>, +} + +/// One entry `sqb load` runs, in execution order. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LoadEntry { + /// An intermediate loader that no selected source runs. + Loader(String), + /// A managed source. + Source(String), +} + +/// How one load ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LoadStatus { + Success, + Skipped, + Failed, +} + +impl LoadStatus { + /// The status from its `ExecutionStatus` value; anything unknown counts as failed. + #[must_use] + pub fn from_value(value: &str) -> Self { + match value { + "success" => Self::Success, + "skipped" => Self::Skipped, + _ => Self::Failed, + } + } +} + +/// One load result as the command renders it. +#[derive(Clone, Debug, PartialEq)] +pub struct LoadResultText { + pub source_name: String, + pub loader_name: String, + /// The `ExecutionResourceKind` value, such as `source` or `loader`. + pub resource_kind: String, + pub status: LoadStatus, + pub rows_loaded: i64, + pub duration_ms: Option, + /// Messages of the result's log lifecycle events. + pub log_messages: Vec, + pub error_message: Option, + pub warning_messages: Vec, +} + +/// Result counts of one load run. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct LoadCounts { + pub pass: i64, + pub warn: i64, + pub fail: i64, + pub skip: i64, + pub total: i64, +} + +/// The source a spinner row shows: position, kind, name and the latest loader message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LoadSpinnerRow { + pub ordinal: i64, + pub total: i64, + pub resource_kind: String, + pub name: String, + pub message: String, +} diff --git a/crates/sqlbuild-core/src/load_command/tests.rs b/crates/sqlbuild-core/src/load_command/tests.rs new file mode 100644 index 0000000000..8c27da4635 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_load_command.rs"] +mod load_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/load_command/tests/helpers.rs b/crates/sqlbuild-core/src/load_command/tests/helpers.rs new file mode 100644 index 0000000000..cca69b32a9 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/tests/helpers.rs @@ -0,0 +1,52 @@ +use crate::load_command::models::{LoadEntry, LoadGraph, LoadResultText, LoadStatus}; + +fn owned(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_owned()).collect() +} + +/// `fetch_events` and `fetch_prices` feed `raw_orders`; `raw_customers` and `raw_regions` stand alone. +pub(super) fn chained_graph() -> LoadGraph { + LoadGraph { + sources: vec![ + ("raw_orders".to_owned(), "raw_orders".to_owned()), + ("raw_customers".to_owned(), "load_customers".to_owned()), + ("raw_regions".to_owned(), "load_customers".to_owned()), + ], + loaders: vec![ + ("fetch_events".to_owned(), Vec::new()), + ("fetch_prices".to_owned(), owned(&["fetch_events"])), + ( + "raw_orders".to_owned(), + owned(&["fetch_events", "fetch_prices"]), + ), + ("load_customers".to_owned(), Vec::new()), + ], + } +} + +pub(super) fn loader(name: &str) -> LoadEntry { + LoadEntry::Loader(name.to_owned()) +} + +pub(super) fn source(name: &str) -> LoadEntry { + LoadEntry::Source(name.to_owned()) +} + +pub(super) fn selectors(names: &[&str]) -> Vec { + owned(names) +} + +/// A finished source load with no logs, warnings or error. +pub(super) fn result(name: &str, status: LoadStatus) -> LoadResultText { + LoadResultText { + source_name: name.to_owned(), + loader_name: name.to_owned(), + resource_kind: "source".to_owned(), + status, + rows_loaded: 0, + duration_ms: None, + log_messages: Vec::new(), + error_message: None, + warning_messages: Vec::new(), + } +} diff --git a/crates/sqlbuild-core/src/load_command/tests/test_load_command.rs b/crates/sqlbuild-core/src/load_command/tests/test_load_command.rs new file mode 100644 index 0000000000..cdb0f54066 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/tests/test_load_command.rs @@ -0,0 +1,199 @@ +use crate::cli_text::models::CliStyle; +use crate::load_command::main::load_counts::load_counts; +use crate::load_command::main::load_exit_code::load_exit_code; +use crate::load_command::main::load_reference_loaders::load_reference_loaders; +use crate::load_command::main::render_load_footer::render_load_footer; +use crate::load_command::main::render_load_result::render_load_result; +use crate::load_command::main::select_load_entries::select_load_entries; +use crate::load_command::models::{LoadResultText, LoadStatus}; +use crate::load_command::tests::helpers::{chained_graph, loader, result, selectors, source}; +use crate::load_command::tests::test_types::{ + FooterTestCase, ReferenceTestCase, ResultLineTestCase, SelectionTestCase, +}; + +#[test] +fn given_selectors_when_selecting_then_orders_loaders_before_sources() { + let test_cases = [ + SelectionTestCase { + description: "no selectors load every source with its upstream loaders", + select: &[], + exclude: &[], + expected_entries: Ok(vec![ + loader("fetch_events"), + loader("fetch_prices"), + source("raw_orders"), + source("raw_customers"), + source("raw_regions"), + ]), + }, + SelectionTestCase { + description: "a bare source runs only its own loader", + select: &["raw_orders"], + exclude: &[], + expected_entries: Ok(vec![source("raw_orders")]), + }, + SelectionTestCase { + description: "a loader name selects all its sources", + select: &["load_customers"], + exclude: &[], + expected_entries: Ok(vec![source("raw_customers"), source("raw_regions")]), + }, + SelectionTestCase { + description: "a trailing plus adds downstream loaders and sources", + select: &[" fetch_prices+ "], + exclude: &[], + expected_entries: Ok(vec![loader("fetch_prices"), source("raw_orders")]), + }, + SelectionTestCase { + description: "excluding a dependency prunes loaders that need it", + select: &[], + exclude: &["fetch_events"], + expected_entries: Ok(vec![source("raw_customers"), source("raw_regions")]), + }, + SelectionTestCase { + description: "excluding one of a loader's sources keeps the loader for the other", + select: &["load_customers"], + exclude: &["raw_regions"], + expected_entries: Ok(vec![source("raw_customers")]), + }, + SelectionTestCase { + description: "an unknown name is rejected", + select: &["+nope"], + exclude: &[], + expected_entries: Err("nope"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + select_load_entries( + &chained_graph(), + (&selectors(test_case.select), &selectors(test_case.exclude)), + ) + .map_err(|error| error.name), + test_case.expected_entries.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selected_sources_when_resolving_references_then_returns_unselected_upstream_loaders() { + let test_cases = [ + ReferenceTestCase { + description: "a terminal source references every upstream loader in order", + selected: vec![("raw_orders".to_owned(), Some("raw_orders".to_owned()))], + expected_loaders: &["fetch_events", "fetch_prices"], + }, + ReferenceTestCase { + description: "selected loaders and sources without loaders are not references", + selected: vec![ + ("fetch_prices".to_owned(), Some("fetch_prices".to_owned())), + ("raw_orders".to_owned(), Some("raw_orders".to_owned())), + ("raw_static".to_owned(), None), + ], + expected_loaders: &["fetch_events"], + }, + ]; + + for test_case in test_cases { + assert_eq!( + load_reference_loaders(&chained_graph(), &test_case.selected), + selectors(test_case.expected_loaders), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_result_when_rendering_then_shows_status_rows_logs_and_error() { + let test_cases = [ + ResultLineTestCase { + description: "a failed loader groups rows and shows its error", + result: LoadResultText { + resource_kind: "loader".to_owned(), + rows_loaded: 1_234_567, + duration_ms: Some(1500.0), + error_message: Some("boom".to_owned()), + ..result("fetch_b", LoadStatus::Failed) + }, + use_color: false, + expected_text: " 2/3 loader fetch_b FAIL 1.50s rows=1,234,567\n boom\n", + }, + ResultLineTestCase { + description: "logs render as dim blocks and colour keeps the status unpadded", + result: LoadResultText { + log_messages: vec!["one\r\ntwo".to_owned()], + ..result("raw_a", LoadStatus::Skipped) + }, + use_color: true, + expected_text: " 2/3 source raw_a \u{1b}[2mSKIP\u{1b}[0m rows=0\n\n\ + \u{1b}[2m log \u{1b}[0m\u{1b}[2mone\u{1b}[0m\n\u{1b}[2m two\u{1b}[0m\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_load_result( + &test_case.result, + (2, 3), + test_case.result.duration_ms, + CliStyle { + use_color: test_case.use_color + }, + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_completing_then_reports_warnings_counts_and_exit_code() { + let test_cases = [ + FooterTestCase { + description: "warnings are listed and make the run complete with warnings", + results: vec![ + LoadResultText { + warning_messages: vec!["[P008] named raw".to_owned()], + loader_name: "load_a".to_owned(), + ..result("raw_a", LoadStatus::Success) + }, + result("raw_b", LoadStatus::Success), + result("raw_c", LoadStatus::Skipped), + ], + expected_text: "\nWarnings:\n\n raw_a (loader load_a)\n [P008] named raw\n\n\ + \u{2713} Completed with warnings PASS=1 WARN=1 FAIL=0 SKIP=1 TOTAL=3 (1.25s)\n", + expected_exit_code: 0, + }, + FooterTestCase { + description: "a failure completes with errors and exits 1", + results: vec![result("raw_a", LoadStatus::Failed)], + expected_text: "\n\u{2717} Completed with errors PASS=0 WARN=0 FAIL=1 SKIP=0 TOTAL=1 (1.25s)\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + let counts = load_counts(&test_case.results); + assert_eq!( + ( + render_load_footer( + &test_case.results, + (counts, 1.25), + CliStyle { use_color: false } + ), + load_exit_code(counts), + ), + ( + test_case.expected_text.to_owned(), + test_case.expected_exit_code + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/load_command/tests/test_types.rs b/crates/sqlbuild-core/src/load_command/tests/test_types.rs new file mode 100644 index 0000000000..31bd8adb72 --- /dev/null +++ b/crates/sqlbuild-core/src/load_command/tests/test_types.rs @@ -0,0 +1,28 @@ +use crate::load_command::models::{LoadEntry, LoadResultText}; + +pub(super) struct SelectionTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) exclude: &'static [&'static str], + pub(super) expected_entries: Result, &'static str>, +} + +pub(super) struct ReferenceTestCase { + pub(super) description: &'static str, + pub(super) selected: Vec<(String, Option)>, + pub(super) expected_loaders: &'static [&'static str], +} + +pub(super) struct ResultLineTestCase { + pub(super) description: &'static str, + pub(super) result: LoadResultText, + pub(super) use_color: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct FooterTestCase { + pub(super) description: &'static str, + pub(super) results: Vec, + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index a715a802dc..4338385c18 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -102,3 +102,9 @@ pub(crate) const ERROR_OUTCOME: &str = "error"; /// Inspection SQL longer than this many characters is cut. pub(crate) const SQL_PREVIEW_LENGTH: usize = 160; pub(crate) const EXECUTION_TITLE: &str = "Execution"; +pub(crate) const LINEAGE_COMMAND: &str = "lineage"; +pub(crate) const QUERY_COMMAND: &str = "query"; +pub(crate) const DBT_COMMAND: &str = "dbt"; +pub(crate) const DBT_JSON_OPTION: &str = "--json"; +pub(crate) const JSON_FORMAT: &str = "json"; +pub(crate) const CSV_FORMAT: &str = "csv"; diff --git a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs new file mode 100644 index 0000000000..e483f76b46 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs @@ -0,0 +1,21 @@ +//! Whether a command's stdout carries machine output, so human progress must use stderr. + +use crate::progress::constants::{ + CSV_FORMAT, DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, QUERY_COMMAND, +}; +use crate::progress::models::OutputModeFacts; + +/// `--json`, `sqb dbt ... --json`, `lineage --format json` or `query --format csv|json`. +#[must_use] +pub fn machine_output_mode(facts: &OutputModeFacts) -> bool { + let format = |value: &Option| value.as_deref().unwrap_or_default().to_owned(); + match facts.command.as_deref() { + _ if facts.json => true, + Some(DBT_COMMAND) => facts.dbt_args.iter().any(|arg| arg == DBT_JSON_OPTION), + Some(LINEAGE_COMMAND) => format(&facts.lineage_format) == JSON_FORMAT, + Some(QUERY_COMMAND) => { + [CSV_FORMAT, JSON_FORMAT].contains(&format(&facts.query_format).as_str()) + } + _ => false, + } +} diff --git a/crates/sqlbuild-core/src/progress/main/mod.rs b/crates/sqlbuild-core/src/progress/main/mod.rs index fdbaea3b79..67bb535223 100644 --- a/crates/sqlbuild-core/src/progress/main/mod.rs +++ b/crates/sqlbuild-core/src/progress/main/mod.rs @@ -9,4 +9,5 @@ pub mod format_expectation_detail; pub mod format_expectation_name; pub mod format_inspection_record; pub mod is_planning_completion_message; +pub mod machine_output_mode; pub mod progress_terminal_duration; diff --git a/crates/sqlbuild-core/src/progress/models.rs b/crates/sqlbuild-core/src/progress/models.rs index 7ec2be6f8f..20abcdde23 100644 --- a/crates/sqlbuild-core/src/progress/models.rs +++ b/crates/sqlbuild-core/src/progress/models.rs @@ -142,3 +142,14 @@ pub struct InspectionRecord { pub row_count: Option, pub error: Option, } + +/// The parsed options that decide whether a command writes machine output to stdout. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct OutputModeFacts { + pub json: bool, + pub command: Option, + pub lineage_format: Option, + pub query_format: Option, + /// The arguments after `sqb dbt `, where `--json` selects JSON output. + pub dbt_args: Vec, +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs index 7660f98bcd..c6c45ce34d 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs @@ -6,12 +6,13 @@ use crate::progress::main::format_expectation_detail::format_expectation_detail; use crate::progress::main::format_expectation_name::format_expectation_name; use crate::progress::main::format_inspection_record::format_inspection_record; use crate::progress::main::is_planning_completion_message::is_planning_completion_message; -use crate::progress::models::{ExpectationStep, InspectionRecord}; +use crate::progress::main::machine_output_mode::machine_output_mode; +use crate::progress::models::{ExpectationStep, InspectionRecord, OutputModeFacts}; const PLAIN: CliStyle = CliStyle { use_color: false }; use crate::progress::tests::test_types::{ CompletionMessageTestCase, ExpectationDetailTestCase, InspectionRecordTestCase, - ProgressLineTestCase, + OutputModeTestCase, ProgressLineTestCase, }; #[test] @@ -198,3 +199,57 @@ fn given_inspection_record_when_formatting_then_sql_is_collapsed_and_cut() { ); } } + +#[test] +fn given_parsed_options_when_choosing_output_mode_then_machine_output_is_detected() { + let facts = |command: &str, dbt_args: &[&str]| OutputModeFacts { + command: Some(command.to_owned()), + dbt_args: dbt_args.iter().map(|arg| (*arg).to_owned()).collect(), + ..OutputModeFacts::default() + }; + let test_cases = [ + OutputModeTestCase { + description: "dbt plan with --json", + facts: facts("dbt", &["--select", "orders", "--json"]), + expected_machine_output: true, + }, + OutputModeTestCase { + description: "dbt plan without --json", + facts: facts("dbt", &["--select", "orders"]), + expected_machine_output: false, + }, + OutputModeTestCase { + description: "plan --json", + facts: OutputModeFacts { + json: true, + ..facts("plan", &[]) + }, + expected_machine_output: true, + }, + OutputModeTestCase { + description: "query csv", + facts: OutputModeFacts { + query_format: Some("csv".to_owned()), + ..facts("query", &[]) + }, + expected_machine_output: true, + }, + OutputModeTestCase { + description: "lineage text", + facts: OutputModeFacts { + lineage_format: Some("text".to_owned()), + ..facts("lineage", &[]) + }, + expected_machine_output: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + machine_output_mode(&test_case.facts), + test_case.expected_machine_output, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_types.rs b/crates/sqlbuild-core/src/progress/tests/test_types.rs index d65c94e451..48942a07ce 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_types.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_types.rs @@ -34,3 +34,9 @@ pub(super) struct ProgressLineTestCase { pub(super) line: String, pub(super) expected_line: &'static str, } + +pub(super) struct OutputModeTestCase { + pub(super) description: &'static str, + pub(super) facts: crate::progress::models::OutputModeFacts, + pub(super) expected_machine_output: bool, +} diff --git a/crates/sqlbuild-core/src/redaction/_helpers/mod.rs b/crates/sqlbuild-core/src/redaction/_helpers/mod.rs new file mode 100644 index 0000000000..d0c4d5d890 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod patterns; diff --git a/crates/sqlbuild-core/src/redaction/_helpers/patterns.rs b/crates/sqlbuild-core/src/redaction/_helpers/patterns.rs new file mode 100644 index 0000000000..69f654ab16 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/_helpers/patterns.rs @@ -0,0 +1,37 @@ +//! The four secret patterns, compiled once per process. + +use std::sync::LazyLock; + +use regex::{Regex, RegexBuilder}; + +use crate::redaction::constants::{ + AUTHORIZATION_PATTERN, BEARER_PATTERN, PATTERN_SIZE_LIMIT, SECRET_VALUE_PATTERN, + URI_CREDENTIAL_PATTERN, +}; + +pub(crate) static AUTHORIZATION: LazyLock> = + LazyLock::new(|| compiled(AUTHORIZATION_PATTERN)); +pub(crate) static BEARER: LazyLock> = + LazyLock::new(|| compiled(BEARER_PATTERN)); +pub(crate) static SECRET_VALUE: LazyLock> = + LazyLock::new(|| compiled(SECRET_VALUE_PATTERN)); +pub(crate) static URI_CREDENTIAL: LazyLock> = + LazyLock::new(|| compiled(URI_CREDENTIAL_PATTERN)); + +fn compiled(pattern: &str) -> Result { + RegexBuilder::new(pattern) + .size_limit(PATTERN_SIZE_LIMIT) + .build() +} + +/// `pattern.sub(replacement, text)`; a pattern that failed to compile matches nothing. +pub(crate) fn substituted( + pattern: &LazyLock>, + text: &str, + replacement: &str, +) -> String { + match pattern.as_ref() { + Ok(regex) => regex.replace_all(text, replacement).into_owned(), + Err(_) => text.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/redaction/constants.rs b/crates/sqlbuild-core/src/redaction/constants.rs new file mode 100644 index 0000000000..6b823c8bb1 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/constants.rs @@ -0,0 +1,33 @@ +//! Secret field names and the replacement diagnostic redaction writes. + +pub(crate) const SECRET_FIELD_PARTS: [&str; 11] = [ + "access_key", + "api_key", + "authorization", + "client_secret", + "credential", + "dsn", + "password", + "private_key", + "refresh_token", + "secret", + "token", +]; +pub(crate) const SECRET_VALUE_PATTERN: &str = concat!( + r"(?i)\b(api[-_]?key|access[-_]?key|private[-_]?key|authorization|", + r"client[-_]?secret|refresh[-_]?token|", + r"password|passwd|pwd|secret|token|dsn|connection_string)", + r#"(\s*[=:]\s*)("[^"\r\n]{0,512}"|'[^'\r\n]{0,512}'|[^\s,;]{1,512})"#, +); +pub(crate) const AUTHORIZATION_PATTERN: &str = concat!( + r"(?i)\b(authorization)(\s*[=:]\s*)(?:basic|bearer)\s+", + r#"("[^"\r\n]{1,512}"|'[^'\r\n]{1,512}'|[^\s,;]{1,512})"#, +); +pub(crate) const BEARER_PATTERN: &str = r"(?i)\bbearer\s+[a-z0-9._~+/=-]{1,512}"; +pub(crate) const URI_CREDENTIAL_PATTERN: &str = + r"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@]+:)([^\s@]+)(@)"; +pub(crate) const PAIR_REPLACEMENT: &str = "${1}${2}[REDACTED]"; +pub(crate) const BEARER_REPLACEMENT: &str = "Bearer [REDACTED]"; +pub(crate) const URI_REPLACEMENT: &str = "${1}[REDACTED]${3}"; +/// Compiled size allowance for the bounded Unicode repetitions above. +pub(crate) const PATTERN_SIZE_LIMIT: usize = 1 << 28; diff --git a/crates/sqlbuild-core/src/redaction/main/mod.rs b/crates/sqlbuild-core/src/redaction/main/mod.rs new file mode 100644 index 0000000000..47aff8ca43 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/main/mod.rs @@ -0,0 +1,2 @@ +pub mod redact_text; +pub mod secret_field_name; diff --git a/crates/sqlbuild-core/src/redaction/main/redact_text.rs b/crates/sqlbuild-core/src/redaction/main/redact_text.rs new file mode 100644 index 0000000000..9764c88a0e --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/main/redact_text.rs @@ -0,0 +1,15 @@ +//! `redact_text`: obvious named secrets and URI passwords removed from diagnostic text. + +use crate::redaction::_helpers::patterns::{ + AUTHORIZATION, BEARER, SECRET_VALUE, URI_CREDENTIAL, substituted, +}; +use crate::redaction::constants::{BEARER_REPLACEMENT, PAIR_REPLACEMENT, URI_REPLACEMENT}; + +/// Authorization headers, bearer tokens, `name=value` secrets, then URI passwords redacted. +#[must_use] +pub fn redact_text(value: &str) -> String { + let authorization: String = substituted(&AUTHORIZATION, value, PAIR_REPLACEMENT); + let bearer: String = substituted(&BEARER, &authorization, BEARER_REPLACEMENT); + let named: String = substituted(&SECRET_VALUE, &bearer, PAIR_REPLACEMENT); + substituted(&URI_CREDENTIAL, &named, URI_REPLACEMENT) +} diff --git a/crates/sqlbuild-core/src/redaction/main/secret_field_name.rs b/crates/sqlbuild-core/src/redaction/main/secret_field_name.rs new file mode 100644 index 0000000000..3895df24be --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/main/secret_field_name.rs @@ -0,0 +1,10 @@ +//! `secret_field_name`: whether a structured logging field holds a secret by its name. + +use crate::redaction::constants::SECRET_FIELD_PARTS; + +/// Whether the lowercased name, with `-` read as `_`, contains a secret field part. +#[must_use] +pub fn secret_field_name(name: &str) -> bool { + let lowered: String = name.to_lowercase().replace('-', "_"); + SECRET_FIELD_PARTS.iter().any(|part| lowered.contains(part)) +} diff --git a/crates/sqlbuild-core/src/redaction/mod.rs b/crates/sqlbuild-core/src/redaction/mod.rs new file mode 100644 index 0000000000..37f9081f4f --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/redaction/tests.rs b/crates/sqlbuild-core/src/redaction/tests.rs new file mode 100644 index 0000000000..bb9efc8856 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_redact_text.rs"] +mod redact_text; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs b/crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs new file mode 100644 index 0000000000..a998e3bc7c --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs @@ -0,0 +1,32 @@ +use crate::redaction::main::redact_text::redact_text; +use crate::redaction::tests::test_types::RedactTestCase; + +#[test] +fn given_diagnostic_text_when_redacting_then_matches_python_substitutions() { + let test_cases = [ + RedactTestCase { + description: "authorization header keeps its scheme separator", + text: "Authorization: Bearer abc.def", + expected_text: "Authorization: [REDACTED]", + }, + RedactTestCase { + description: "named secrets and URI passwords", + text: "password = 'hunter2', url=postgres://user:pw@host/db", + expected_text: "password = [REDACTED], url=postgres://user:[REDACTED]@host/db", + }, + RedactTestCase { + description: "plain text is unchanged", + text: "nothing to see", + expected_text: "nothing to see", + }, + ]; + + for test_case in test_cases { + assert_eq!( + redact_text(test_case.text), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/redaction/tests/test_types.rs b/crates/sqlbuild-core/src/redaction/tests/test_types.rs new file mode 100644 index 0000000000..7466e971e7 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(crate) struct RedactTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/seed_command/constants.rs b/crates/sqlbuild-core/src/seed_command/constants.rs new file mode 100644 index 0000000000..ea0d36d5b3 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/constants.rs @@ -0,0 +1,6 @@ +//! Seed output labels and widths. + +pub(crate) const OK_STATUS: &str = "OK"; +pub(crate) const FAIL_STATUS: &str = "FAIL"; +pub(crate) const SEED_NAME_WIDTH: usize = 30; +pub(crate) const STATUS_WIDTH: usize = 6; diff --git a/crates/sqlbuild-core/src/seed_command/main/mod.rs b/crates/sqlbuild-core/src/seed_command/main/mod.rs new file mode 100644 index 0000000000..b217f85970 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/main/mod.rs @@ -0,0 +1,4 @@ +pub mod render_seed_completion; +pub mod render_seed_ready; +pub mod render_seed_row; +pub mod seed_exit_code; diff --git a/crates/sqlbuild-core/src/seed_command/main/render_seed_completion.rs b/crates/sqlbuild-core/src/seed_command/main/render_seed_completion.rs new file mode 100644 index 0000000000..501c94a016 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/main/render_seed_completion.rs @@ -0,0 +1,41 @@ +//! The seed completion line. + +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, CompletionState, SummaryCount}; + +fn count(label: &str, value: usize) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value: i64::try_from(value).unwrap_or(i64::MAX), + } +} + +/// A blank line and the completion state with PASS, WARN, FAIL, SKIP and TOTAL counts. +#[must_use] +pub fn render_seed_completion( + (passed, failed, total): (usize, usize, usize), + elapsed_seconds: f64, + style: CliStyle, +) -> String { + let summary: String = format_summary_footer( + style, + &[ + count("PASS", passed), + count("WARN", 0), + count("FAIL", failed), + count("SKIP", 0), + count("TOTAL", total), + ], + Some(&format!("{elapsed_seconds:.2}s")), + ); + let (state, label) = if failed == 0 { + (CompletionState::Ok, "Completed successfully") + } else { + (CompletionState::Fail, "Completed with errors") + }; + format!( + "\n{}\n", + format_completion_line(style, state, label, Some(&summary)) + ) +} diff --git a/crates/sqlbuild-core/src/seed_command/main/render_seed_ready.rs b/crates/sqlbuild-core/src/seed_command/main/render_seed_ready.rs new file mode 100644 index 0000000000..70e258ed4d --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/main/render_seed_ready.rs @@ -0,0 +1,24 @@ +//! The header announcing the selected seeds. + +use crate::cli_text::main::format_surface_header::format_surface_header; +use crate::cli_text::main::format_tree_connector::format_tree_connector; +use crate::cli_text::models::CliStyle; + +/// `Seed ready selected`, then the `Seeds ()` tree and a blank line. +#[must_use] +pub fn render_seed_ready(names: &[String], style: CliStyle) -> String { + let count: usize = names.len(); + let mut text: String = format!( + "\n{}\n\n{}\n", + format_surface_header(style, "Seed ready", &format!("{count} selected")), + style.section(&format!("Seeds ({count})")) + ); + for (index, name) in names.iter().enumerate() { + text.push_str(&format!( + "{} {name}\n", + format_tree_connector(style, index + 1 == count) + )); + } + text.push('\n'); + text +} diff --git a/crates/sqlbuild-core/src/seed_command/main/render_seed_row.rs b/crates/sqlbuild-core/src/seed_command/main/render_seed_row.rs new file mode 100644 index 0000000000..b6b9337b4b --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/main/render_seed_row.rs @@ -0,0 +1,34 @@ +//! The progress row of one finished seed. + +use crate::cli_text::main::format_coded_error::format_coded_error; +use crate::cli_text::models::CliStyle; +use crate::seed_command::constants::{FAIL_STATUS, OK_STATUS, SEED_NAME_WIDTH, STATUS_WIDTH}; +use crate::seed_command::models::SeedRow; + +/// ` / seed `, then its error line when there is one. +#[must_use] +pub fn render_seed_row(row: &SeedRow, style: CliStyle) -> String { + let status: String = style.status(if row.succeeded { + OK_STATUS + } else { + FAIL_STATUS + }); + let duration: String = row + .duration_ms + .map(|milliseconds| format!("{:.2}s", milliseconds / 1000.0)) + .unwrap_or_default(); + let mut text: String = format!( + " {}/{} seed {: { + format_coded_error((code, message, row.error_help.as_deref()), true, style) + } + None => message.clone(), + }; + text.push_str(&format!(" {rendered}\n")); + } + text +} diff --git a/crates/sqlbuild-core/src/seed_command/main/seed_exit_code.rs b/crates/sqlbuild-core/src/seed_command/main/seed_exit_code.rs new file mode 100644 index 0000000000..66d0312466 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/main/seed_exit_code.rs @@ -0,0 +1,7 @@ +//! The exit code of `sqb seed`. + +/// 1 when any seed failed, else 0. +#[must_use] +pub fn seed_exit_code(failed: usize) -> i32 { + i32::from(failed > 0) +} diff --git a/crates/sqlbuild-core/src/seed_command/mod.rs b/crates/sqlbuild-core/src/seed_command/mod.rs new file mode 100644 index 0000000000..b87e640489 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/mod.rs @@ -0,0 +1,8 @@ +//! `sqb seed`: the ready header, per-seed rows, the completion summary and the exit code. + +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/seed_command/models.rs b/crates/sqlbuild-core/src/seed_command/models.rs new file mode 100644 index 0000000000..f28258b8f9 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/models.rs @@ -0,0 +1,14 @@ +//! One finished seed as its progress row shows it. + +/// A completed seed load. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct SeedRow { + pub name: String, + pub ordinal: usize, + pub total: usize, + pub succeeded: bool, + pub duration_ms: Option, + pub error_message: Option, + pub error_code: Option, + pub error_help: Option, +} diff --git a/crates/sqlbuild-core/src/seed_command/tests.rs b/crates/sqlbuild-core/src/seed_command/tests.rs new file mode 100644 index 0000000000..c08eaa61e2 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_seed_command.rs"] +mod seed_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/seed_command/tests/test_seed_command.rs b/crates/sqlbuild-core/src/seed_command/tests/test_seed_command.rs new file mode 100644 index 0000000000..b9b538f11f --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/tests/test_seed_command.rs @@ -0,0 +1,119 @@ +use crate::cli_text::models::CliStyle; +use crate::seed_command::main::render_seed_completion::render_seed_completion; +use crate::seed_command::main::render_seed_ready::render_seed_ready; +use crate::seed_command::main::render_seed_row::render_seed_row; +use crate::seed_command::main::seed_exit_code::seed_exit_code; +use crate::seed_command::models::SeedRow; +use crate::seed_command::tests::test_types::{CompletionTestCase, ReadyTestCase, RowTestCase}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_selected_seeds_when_rendering_ready_then_lists_them_as_a_tree() { + let test_cases = [ + ReadyTestCase { + description: "two seeds end with the last branch", + names: vec!["first_seed".to_owned(), "second_seed".to_owned()], + expected_text: "\nSeed ready 2 selected\n\nSeeds (2)\n├── first_seed\n└── second_seed\n\n", + }, + ReadyTestCase { + description: "no seeds keep the header", + names: Vec::new(), + expected_text: "\nSeed ready 0 selected\n\nSeeds (0)\n\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_seed_ready(&test_case.names, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_finished_seed_when_rendering_row_then_pads_name_and_shows_error() { + let test_cases = [ + RowTestCase { + description: "a success shows its duration", + row: SeedRow { + name: "first_seed".to_owned(), + ordinal: 1, + total: 2, + succeeded: true, + duration_ms: Some(1234.0), + ..SeedRow::default() + }, + expected_text: " 1/2 seed first_seed OK 1.23s\n", + }, + RowTestCase { + description: "a coded failure renders the labelled error and help", + row: SeedRow { + name: "bad_seed".to_owned(), + ordinal: 2, + total: 2, + error_message: Some("bad value\ndetail".to_owned()), + error_code: Some("S001".to_owned()), + error_help: Some("fix the CSV".to_owned()), + ..SeedRow::default() + }, + expected_text: " 2/2 seed bad_seed FAIL \n error[S001]: bad value\ndetail\n = help: fix the CSV\n", + }, + RowTestCase { + description: "an uncoded failure shows the message", + row: SeedRow { + name: "bad_seed".to_owned(), + ordinal: 1, + total: 1, + duration_ms: Some(5.0), + error_message: Some("boom".to_owned()), + ..SeedRow::default() + }, + expected_text: " 1/1 seed bad_seed FAIL 0.01s\n boom\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_seed_row(&test_case.row, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_completing_then_summarizes_and_resolves_exit_code() { + let test_cases = [ + CompletionTestCase { + description: "all seeds loaded", + counts: (2, 0, 2), + expected_text: "\n✓ Completed successfully PASS=2 WARN=0 FAIL=0 SKIP=0 TOTAL=2 (1.50s)\n", + expected_exit_code: 0, + }, + CompletionTestCase { + description: "a failed seed fails the run", + counts: (1, 1, 2), + expected_text: "\n✗ Completed with errors PASS=1 WARN=0 FAIL=1 SKIP=0 TOTAL=2 (1.50s)\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_seed_completion(test_case.counts, 1.5, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + assert_eq!( + seed_exit_code(test_case.counts.1), + test_case.expected_exit_code, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/seed_command/tests/test_types.rs b/crates/sqlbuild-core/src/seed_command/tests/test_types.rs new file mode 100644 index 0000000000..278331c6b7 --- /dev/null +++ b/crates/sqlbuild-core/src/seed_command/tests/test_types.rs @@ -0,0 +1,20 @@ +use crate::seed_command::models::SeedRow; + +pub(super) struct ReadyTestCase { + pub(super) description: &'static str, + pub(super) names: Vec, + pub(super) expected_text: &'static str, +} + +pub(super) struct RowTestCase { + pub(super) description: &'static str, + pub(super) row: SeedRow, + pub(super) expected_text: &'static str, +} + +pub(super) struct CompletionTestCase { + pub(super) description: &'static str, + pub(super) counts: (usize, usize, usize), + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} diff --git a/crates/sqlbuild-core/src/test_command/constants.rs b/crates/sqlbuild-core/src/test_command/constants.rs new file mode 100644 index 0000000000..883fcc2cf2 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/constants.rs @@ -0,0 +1,9 @@ +//! Test output labels and widths. + +pub(crate) const ERROR_LABEL: &str = "error"; +pub(crate) const ERROR_LABEL_WIDTH: usize = 10; +pub(crate) const DEFAULT_ERROR: &str = "test failed"; +pub(crate) const UNKNOWN_GROUP: &str = "(unknown)"; +pub(crate) const NAME_MIN_WIDTH: usize = 50; +pub(crate) const CHILD_NAME_INDENT: usize = 4; +pub(crate) const ERROR_SEVERITY: &str = "error"; diff --git a/crates/sqlbuild-core/src/test_command/main/mod.rs b/crates/sqlbuild-core/src/test_command/main/mod.rs new file mode 100644 index 0000000000..55ae286b69 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/mod.rs @@ -0,0 +1,8 @@ +pub mod parameterized_test_label; +pub mod render_test_completion; +pub mod render_test_inspection; +pub mod render_test_ready; +pub mod test_exit_code; +pub mod test_group_name; +pub mod test_name_width; +pub mod test_outcome_status; diff --git a/crates/sqlbuild-core/src/test_command/main/parameterized_test_label.rs b/crates/sqlbuild-core/src/test_command/main/parameterized_test_label.rs new file mode 100644 index 0000000000..5728eab8f4 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/parameterized_test_label.rs @@ -0,0 +1,31 @@ +//! The label of a test row. + +use crate::test_command::models::TestParameter; + +/// The name, with source path and typed values only for parameterized cases. +#[must_use] +pub fn parameterized_test_label( + name: &str, + source_path: Option<&str>, + parameters: &[TestParameter], +) -> String { + let Some(source_path) = source_path else { + return name.to_owned(); + }; + if parameters.is_empty() { + return name.to_owned(); + } + let rendered: Vec = parameters + .iter() + .map(|parameter| { + format!( + "{}:{}{}={}", + parameter.name, + parameter.value_type, + if parameter.nullable { "?" } else { "" }, + parameter.rendered + ) + }) + .collect(); + format!("{name} ({source_path}; {})", rendered.join(", ")) +} diff --git a/crates/sqlbuild-core/src/test_command/main/render_test_completion.rs b/crates/sqlbuild-core/src/test_command/main/render_test_completion.rs new file mode 100644 index 0000000000..fcf1d9decd --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/render_test_completion.rs @@ -0,0 +1,72 @@ +//! The test summary footer and failure details. + +use crate::cli_text::main::format_coded_error::format_coded_error; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, SummaryCount}; +use crate::test_command::constants::{DEFAULT_ERROR, ERROR_LABEL, ERROR_LABEL_WIDTH}; +use crate::test_command::models::TestFailure; +use crate::text::main::python_splitlines::python_splitlines; + +fn count(label: &str, value: usize) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value: i64::try_from(value).unwrap_or(i64::MAX), + } +} + +fn failure_lines(failure: &TestFailure, style: CliStyle) -> Vec { + let message: &str = failure + .error_message + .as_deref() + .filter(|text| !text.is_empty()) + .unwrap_or(DEFAULT_ERROR); + let rendered: String = match &failure.error_code { + Some(code) => { + format_coded_error((code, message, failure.error_help.as_deref()), false, style) + } + None => message.to_owned(), + }; + let mut split: Vec<&str> = python_splitlines(&rendered); + if split.is_empty() { + split.push(&rendered); + } + let mut lines: Vec = vec![format!(" {}", failure.label)]; + let label: String = style.error_muted(&format!("{ERROR_LABEL: String { + let summary: String = format_summary_footer( + style, + &[ + count("PASS", passed), + count("FAIL", total - passed), + count("TOTAL", total), + ], + None, + ); + let mut rendered: String = format!("\n{summary}"); + if !failures.is_empty() { + let mut lines: Vec = vec![style.error_strong("Failures:"), String::new()]; + for failure in failures { + lines.extend(failure_lines(failure, style)); + } + rendered = format!("{rendered}\n\n{}", lines.join("\n").trim_end()); + } + format!("{rendered}\n") +} diff --git a/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs b/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs new file mode 100644 index 0000000000..c79d505b15 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs @@ -0,0 +1,55 @@ +//! `sqb test --inspect`: the resolved test plan without executing it. + +use crate::test_command::constants::ERROR_SEVERITY; +use crate::test_command::models::{TestInspectionEntry, TestPlanWarning}; + +fn names(label: &str, values: &[String]) -> String { + let shown: String = if values.is_empty() { + "none".to_owned() + } else { + values.join(", ") + }; + format!(" {label}: {shown}\n") +} + +/// Each test's fixture boundaries and model chain, warnings, the outcome line, and the exit code. +#[must_use] +pub fn render_test_inspection( + entries: &[TestInspectionEntry], + warnings: &[TestPlanWarning], +) -> (String, i32) { + let mut text: String = "\nResolved test plan\n\n".to_owned(); + for entry in entries { + text.push_str(&format!("{}\n", entry.label)); + for (label, values) in [ + ("mocked sources", &entry.mocked_sources), + ("mocked refs", &entry.mocked_refs), + ("mocked seeds", &entry.mocked_seeds), + ("mocked dbt refs", &entry.mocked_dbt_refs), + ("mocked table functions", &entry.mocked_table_functions), + ("real models", &entry.real_models), + ("expected models", &entry.expected_models), + ] { + text.push_str(&names(label, values)); + } + for mock in &entry.mocked_refs { + text.push_str(&format!( + " boundary: {mock} is replaced by __ref__{mock}\n" + )); + } + text.push('\n'); + } + let errors: usize = warnings + .iter() + .filter(|warning| warning.severity == ERROR_SEVERITY) + .count(); + for warning in warnings { + text.push_str(&format!(" {}: {}\n", warning.severity, warning.message)); + } + text.push_str(&format!( + "Test plan inspection {}: {} selected, {errors} errors.\n", + if errors > 0 { "failed" } else { "complete" }, + entries.len() + )); + (text, i32::from(errors > 0)) +} diff --git a/crates/sqlbuild-core/src/test_command/main/render_test_ready.rs b/crates/sqlbuild-core/src/test_command/main/render_test_ready.rs new file mode 100644 index 0000000000..4f248bd388 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/render_test_ready.rs @@ -0,0 +1,17 @@ +//! The header announcing the selected tests. + +use crate::cli_text::main::format_surface_header::format_surface_header; +use crate::cli_text::models::CliStyle; + +/// A blank line, `Test ready selected, models`, and a blank line. +#[must_use] +pub fn render_test_ready((tests, models): (usize, usize), style: CliStyle) -> String { + format!( + "\n{}\n\n", + format_surface_header( + style, + "Test ready", + &format!("{tests} selected, {models} models") + ) + ) +} diff --git a/crates/sqlbuild-core/src/test_command/main/test_exit_code.rs b/crates/sqlbuild-core/src/test_command/main/test_exit_code.rs new file mode 100644 index 0000000000..601794de1b --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/test_exit_code.rs @@ -0,0 +1,7 @@ +//! The exit code of `sqb test`. + +/// 1 when any test did not pass, else 0. +#[must_use] +pub fn test_exit_code(passed: usize, total: usize) -> i32 { + i32::from(passed != total) +} diff --git a/crates/sqlbuild-core/src/test_command/main/test_group_name.rs b/crates/sqlbuild-core/src/test_command/main/test_group_name.rs new file mode 100644 index 0000000000..a6fdb63b18 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/test_group_name.rs @@ -0,0 +1,13 @@ +//! The model a test's progress rows are grouped under. + +use crate::test_command::constants::UNKNOWN_GROUP; + +/// The first model with expected rows, else the first model, else `(unknown)`. +#[must_use] +pub fn test_group_name(chain: &[(String, bool)]) -> String { + chain + .iter() + .find(|(_, expected)| *expected) + .or_else(|| chain.first()) + .map_or_else(|| UNKNOWN_GROUP.to_owned(), |(model, _)| model.clone()) +} diff --git a/crates/sqlbuild-core/src/test_command/main/test_name_width.rs b/crates/sqlbuild-core/src/test_command/main/test_name_width.rs new file mode 100644 index 0000000000..94a73a5192 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/test_name_width.rs @@ -0,0 +1,15 @@ +//! One shared width for test rows and their nested expectation rows. + +use crate::cli_text::main::resolve_name_column_width::resolve_name_column_width; +use crate::test_command::constants::{CHILD_NAME_INDENT, NAME_MIN_WIDTH}; + +/// The test label column width, widened for the longest indented child name. +#[must_use] +pub fn test_name_width(labels: &[String], child_names: &[String]) -> usize { + let child: usize = child_names + .iter() + .map(|name| name.chars().count()) + .max() + .unwrap_or(0); + resolve_name_column_width(labels, NAME_MIN_WIDTH).max(child + CHILD_NAME_INDENT) +} diff --git a/crates/sqlbuild-core/src/test_command/main/test_outcome_status.rs b/crates/sqlbuild-core/src/test_command/main/test_outcome_status.rs new file mode 100644 index 0000000000..42941e1198 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/main/test_outcome_status.rs @@ -0,0 +1,11 @@ +//! The status word of a test outcome. + +/// `PASS`, `ERROR` for execution errors, else `FAIL`. +#[must_use] +pub fn test_outcome_status(outcome: &str) -> &'static str { + match outcome { + "pass" => "PASS", + "error" => "ERROR", + _ => "FAIL", + } +} diff --git a/crates/sqlbuild-core/src/test_command/mod.rs b/crates/sqlbuild-core/src/test_command/mod.rs new file mode 100644 index 0000000000..8c866e79cf --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/mod.rs @@ -0,0 +1,8 @@ +//! `sqb test`: test labels, plan inspection, completion summary and exit code. + +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/test_command/models.rs b/crates/sqlbuild-core/src/test_command/models.rs new file mode 100644 index 0000000000..aa17cc236e --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/models.rs @@ -0,0 +1,40 @@ +//! SQL test plan entries and results as the command renders them. + +/// One typed parameter value of a parameterized test case. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TestParameter { + pub name: String, + pub value_type: String, + pub nullable: bool, + /// The value as compact ASCII JSON. + pub rendered: String, +} + +/// One selected test as `--inspect` shows it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestInspectionEntry { + pub label: String, + pub mocked_sources: Vec, + pub mocked_refs: Vec, + pub mocked_seeds: Vec, + pub mocked_dbt_refs: Vec, + pub mocked_table_functions: Vec, + pub real_models: Vec, + pub expected_models: Vec, +} + +/// One plan warning: its severity value and message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TestPlanWarning { + pub severity: String, + pub message: String, +} + +/// One test that did not pass. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestFailure { + pub label: String, + pub error_message: Option, + pub error_code: Option, + pub error_help: Option, +} diff --git a/crates/sqlbuild-core/src/test_command/tests.rs b/crates/sqlbuild-core/src/test_command/tests.rs new file mode 100644 index 0000000000..5d8555f9fb --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_test_command.rs"] +mod test_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/test_command/tests/helpers.rs b/crates/sqlbuild-core/src/test_command/tests/helpers.rs new file mode 100644 index 0000000000..dfabe0753f --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/tests/helpers.rs @@ -0,0 +1,11 @@ +use crate::test_command::models::TestParameter; + +/// A string parameter with its rendered JSON value. +pub(super) fn parameter(name: &str, nullable: bool, rendered: &str) -> TestParameter { + TestParameter { + name: name.to_owned(), + value_type: "string".to_owned(), + nullable, + rendered: rendered.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs b/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs new file mode 100644 index 0000000000..46b2e84661 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs @@ -0,0 +1,258 @@ +use crate::cli_text::models::CliStyle; +use crate::test_command::main::parameterized_test_label::parameterized_test_label; +use crate::test_command::main::render_test_completion::render_test_completion; +use crate::test_command::main::render_test_inspection::render_test_inspection; +use crate::test_command::main::render_test_ready::render_test_ready; +use crate::test_command::main::test_exit_code::test_exit_code; +use crate::test_command::main::test_group_name::test_group_name; +use crate::test_command::main::test_name_width::test_name_width; +use crate::test_command::main::test_outcome_status::test_outcome_status; +use crate::test_command::models::{TestFailure, TestInspectionEntry, TestPlanWarning}; +use crate::test_command::tests::helpers::parameter; +use crate::test_command::tests::test_types::{ + CompletionTestCase, GroupTestCase, InspectionTestCase, LabelTestCase, OutcomeTestCase, + ReadyTestCase, WidthTestCase, +}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_test_when_labelling_then_adds_source_and_values_only_for_parameterized_cases() { + let test_cases = [ + LabelTestCase { + description: "a plain test keeps its name", + source_path: Some("tests/unit/test_orders.sql"), + parameters: Vec::new(), + expected_label: "order_status", + }, + LabelTestCase { + description: "no source path keeps the name", + source_path: None, + parameters: vec![parameter("status", false, "\"open\"")], + expected_label: "order_status", + }, + LabelTestCase { + description: "parameters render typed values and nullability", + source_path: Some("tests/unit/test_orders.sql"), + parameters: vec![ + parameter("status", false, "\"open\""), + parameter("note", true, "null"), + ], + expected_label: "order_status (tests/unit/test_orders.sql; status:string=\"open\", note:string?=null)", + }, + ]; + + for test_case in test_cases { + assert_eq!( + parameterized_test_label("order_status", test_case.source_path, &test_case.parameters), + test_case.expected_label, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_chain_when_grouping_then_prefers_first_expected_model() { + let test_cases = [ + GroupTestCase { + description: "the first model with expected rows names the group", + chain: vec![("stg".to_owned(), false), ("orders".to_owned(), true)], + expected_group: "orders", + }, + GroupTestCase { + description: "without expected rows the first model names the group", + chain: vec![("stg".to_owned(), false), ("orders".to_owned(), false)], + expected_group: "stg", + }, + GroupTestCase { + description: "an empty chain is unknown", + chain: Vec::new(), + expected_group: "(unknown)", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_group_name(&test_case.chain), + test_case.expected_group, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_outcomes_when_rendering_status_then_distinguishes_errors_from_failures() { + let test_cases = [ + OutcomeTestCase { + description: "a pass is PASS", + outcome: "pass", + expected_status: "PASS", + }, + OutcomeTestCase { + description: "an execution error is ERROR", + outcome: "error", + expected_status: "ERROR", + }, + OutcomeTestCase { + description: "an assertion failure is FAIL", + outcome: "fail", + expected_status: "FAIL", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_outcome_status(test_case.outcome), + test_case.expected_status, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_names_when_resolving_width_then_widens_for_indented_children() { + let test_cases = [ + WidthTestCase { + description: "short names use the minimum width", + labels: vec!["orders_assert".to_owned()], + child_names: Vec::new(), + expected_width: 50, + }, + WidthTestCase { + description: "a long child name widens by its indent", + labels: vec!["orders_assert".to_owned()], + child_names: vec!["x".repeat(60)], + expected_width: 64, + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_name_width(&test_case.labels, &test_case.child_names), + test_case.expected_width, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_counts_when_rendering_ready_then_writes_surface_header() { + let test_cases = [ReadyTestCase { + description: "counts tests and models", + counts: (1, 2), + expected_text: "\nTest ready 1 selected, 2 models\n\n", + }]; + + for test_case in test_cases { + assert_eq!( + render_test_ready(test_case.counts, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_completing_then_renders_summary_failures_and_exit_code() { + let test_cases = [ + CompletionTestCase { + description: "all passing writes only the summary", + counts: (2, 2), + failures: Vec::new(), + expected_text: "\nPASS=2 FAIL=0 TOTAL=2\n", + expected_exit_code: 0, + }, + CompletionTestCase { + description: "a coded failure renders its code, muted lines and help", + counts: (0, 1), + failures: vec![TestFailure { + label: "orders_assert".to_owned(), + error_message: Some("first\nsecond".to_owned()), + error_code: Some("T003".to_owned()), + error_help: Some("fix it".to_owned()), + }], + expected_text: "\nPASS=0 FAIL=1 TOTAL=1\n\nFailures:\n\n orders_assert\n error [T003] first\n second\n = help: fix it\n", + expected_exit_code: 1, + }, + CompletionTestCase { + description: "an uncoded empty failure says the test failed", + counts: (1, 2), + failures: vec![TestFailure { + label: "orders_unique".to_owned(), + error_message: Some(String::new()), + ..TestFailure::default() + }], + expected_text: "\nPASS=1 FAIL=1 TOTAL=2\n\nFailures:\n\n orders_unique\n error test failed\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_test_completion(test_case.counts, &test_case.failures, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + assert_eq!( + test_exit_code(test_case.counts.0, test_case.counts.1), + test_case.expected_exit_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_plan_when_inspecting_then_lists_boundaries_and_counts_errors() { + let entry = TestInspectionEntry { + label: "order_status [open_case]".to_owned(), + mocked_sources: vec!["raw_orders".to_owned()], + mocked_refs: vec!["stg".to_owned()], + real_models: vec!["orders".to_owned()], + expected_models: vec!["orders".to_owned()], + ..TestInspectionEntry::default() + }; + let test_cases = [ + InspectionTestCase { + description: "nothing selected completes", + entries: Vec::new(), + warnings: Vec::new(), + expected_text: "\nResolved test plan\n\nTest plan inspection complete: 0 selected, 0 errors.\n", + expected_exit_code: 0, + }, + InspectionTestCase { + description: "an error warning fails the inspection", + entries: vec![entry], + warnings: vec![ + TestPlanWarning { + severity: "warning".to_owned(), + message: "careful".to_owned(), + }, + TestPlanWarning { + severity: "error".to_owned(), + message: "broken".to_owned(), + }, + ], + expected_text: "\nResolved test plan\n\norder_status [open_case]\n mocked sources: raw_orders\n mocked refs: stg\n mocked seeds: none\n mocked dbt refs: none\n mocked table functions: none\n real models: orders\n expected models: orders\n boundary: stg is replaced by __ref__stg\n\n warning: careful\n error: broken\nTest plan inspection failed: 1 selected, 1 errors.\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_test_inspection(&test_case.entries, &test_case.warnings), + ( + test_case.expected_text.to_owned(), + test_case.expected_exit_code + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/test_command/tests/test_types.rs b/crates/sqlbuild-core/src/test_command/tests/test_types.rs new file mode 100644 index 0000000000..96c173c1d7 --- /dev/null +++ b/crates/sqlbuild-core/src/test_command/tests/test_types.rs @@ -0,0 +1,51 @@ +use crate::test_command::models::{ + TestFailure, TestInspectionEntry, TestParameter, TestPlanWarning, +}; + +pub(super) struct LabelTestCase { + pub(super) description: &'static str, + pub(super) source_path: Option<&'static str>, + pub(super) parameters: Vec, + pub(super) expected_label: &'static str, +} + +pub(super) struct GroupTestCase { + pub(super) description: &'static str, + pub(super) chain: Vec<(String, bool)>, + pub(super) expected_group: &'static str, +} + +pub(super) struct CompletionTestCase { + pub(super) description: &'static str, + pub(super) counts: (usize, usize), + pub(super) failures: Vec, + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} + +pub(super) struct InspectionTestCase { + pub(super) description: &'static str, + pub(super) entries: Vec, + pub(super) warnings: Vec, + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} + +pub(super) struct OutcomeTestCase { + pub(super) description: &'static str, + pub(super) outcome: &'static str, + pub(super) expected_status: &'static str, +} + +pub(super) struct WidthTestCase { + pub(super) description: &'static str, + pub(super) labels: Vec, + pub(super) child_names: Vec, + pub(super) expected_width: usize, +} + +pub(super) struct ReadyTestCase { + pub(super) description: &'static str, + pub(super) counts: (usize, usize), + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index fc2f00f2b3..0ac50aadbd 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -36,6 +36,7 @@ sqlbuild-render.workspace = true sqlbuild-rules.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true +sqlbuild-state.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs new file mode 100644 index 0000000000..a53c860a6b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -0,0 +1,754 @@ +//! `sqb check`: run the selected Python checks after their Python dependencies. + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyInt, PyString, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::check_command::errors::CheckSelectionError; +use sqlbuild_core::check_command::main::check_exit_code::check_exit_code; +use sqlbuild_core::check_command::main::render_check_summary::render_check_summary; +use sqlbuild_core::check_command::main::validate_check_selection::validate_check_selection; +use sqlbuild_core::check_command::models::CheckResultText; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::check::check_results::{ + check_result_text, results_json, write_results, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; +const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main.static_command"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const RELATION_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relation_plan"; +const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; +const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main.graph"; +const PYTHON_SELECTORS_MODULE: &str = "sqlbuild.compiler.python_nodes.main.selectors"; +const PYTHON_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main.run_lifecycle"; +const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; +const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; +const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; +const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main.checks"; +const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main.ingress"; +const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; +const RUNTIME_TARGET_MODULE: &str = + "sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target"; +const GRAPH_OPERATOR_CODE: &str = "C681"; +const GRAPH_OPERATOR_MESSAGE: &str = "sqb check selectors do not support graph operators; \ + select checks directly without '+', '~', or path-between expansion"; +const EXECUTION_COMMAND: &str = "sqb check"; + +/// The compiled pipeline and everything checks run with. +struct Prepared<'py> { + pipeline: Bound<'py, PyAny>, + python_graph: Bound<'py, PyAny>, + check_functions: Bound<'py, PyAny>, + lifecycle_plan: Bound<'py, PyAny>, + relation_targets: Bound<'py, PyAny>, + relation_refs: Bound<'py, PyAny>, + defaults: (Bound<'py, PyAny>, Bound<'py, PyAny>), + project_relations: Bound<'py, PyAny>, +} + +fn user_error(py: Python<'_>, (code, message): (&str, &str)) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", code)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn selection_error(py: Python<'_>, error: &CheckSelectionError) -> PyErr { + user_error(py, (error.code, &error.message)) +} + +fn empty_tuple(py: Python<'_>) -> Bound<'_, PyAny> { + PyTuple::empty(py).into_any() +} + +/// Refuse `+`, `~` and path selectors, splitting selectors as Python's `split` does. +fn refuse_graph_selectors(py: Python<'_>, selectors: &[Bound<'_, PyAny>]) -> PyResult<()> { + let parse: Bound<'_, PyAny> = imported(py, SELECTOR_PARSE_MODULE, "parse_project_selector")?; + let path_selector: Bound<'_, PyAny> = imported(py, PLANNER_MODELS, "PathSelector")?; + for selector in selectors { + for token in selector.call_method0("split")?.try_iter()? { + for part in token?.call_method1("split", (",",))?.try_iter()? { + let parsed: Bound<'_, PyAny> = parse.call1((part?,))?; + if parsed.is_instance(&path_selector)? + || parsed.getattr("upstream")?.is_truthy()? + || parsed.getattr("downstream")?.is_truthy()? + { + return Err(user_error( + py, + (GRAPH_OPERATOR_CODE, GRAPH_OPERATOR_MESSAGE), + )); + } + } + } + } + Ok(()) +} + +fn python_nodes<'py>( + graph: &Bound<'py, PyAny>, + (select, exclude): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + call_with( + &imported( + graph.py(), + PYTHON_SELECTORS_MODULE, + "resolve_python_nodes_from_selectors", + )?, + &[ + ("select", select), + ("exclude", exclude), + ("graph", graph.clone()), + ], + ) +} + +fn node_kind<'py>( + graph: &Bound<'py, PyAny>, + name: &Bound<'py, PyAny>, +) -> PyResult> { + graph + .getattr("nodes_by_name")? + .get_item(name)? + .getattr("kind") +} + +/// The selected check names: selected checks, or every check minus excluded nodes. +fn selected_check_names<'py>( + request: &Bound<'py, PyAny>, + graph: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = graph.py(); + let select: Bound<'py, PyAny> = request.getattr("select")?; + let exclude: Bound<'py, PyAny> = request.getattr("exclude")?; + let selectors: Vec> = select + .try_iter()? + .chain(exclude.try_iter()?) + .collect::>()?; + refuse_graph_selectors(py, &selectors)?; + let check_kind: Bound<'py, PyAny> = + imported(py, PYTHON_NODE_TYPES, "PythonNodeKind")?.getattr("CHECK")?; + if select.is_truthy()? { + let selected: Bound<'py, PyAny> = python_nodes(graph, (select, exclude))?; + let mut non_checks: Vec = Vec::new(); + for name in selected.try_iter()? { + let name: Bound<'py, PyAny> = name?; + if !node_kind(graph, &name)?.eq(&check_kind)? { + non_checks.push(name.extract()?); + } + } + validate_check_selection(&non_checks).map_err(|error| selection_error(py, &error))?; + return selected.downcast_into().map_err(Into::into); + } + let excluded: Bound<'py, PyAny> = if exclude.is_truthy()? { + python_nodes(graph, (exclude, empty_tuple(py)))? + } else { + PyFrozenSet::empty(py)?.into_any() + }; + let mut names: Vec> = Vec::new(); + for node in graph.getattr("nodes")?.try_iter()? { + let node: Bound<'py, PyAny> = node?; + let name: Bound<'py, PyAny> = node.getattr("name")?; + if node.getattr("kind")?.eq(&check_kind)? && !excluded.contains(&name)? { + names.push(name); + } + } + PyFrozenSet::new(py, names) +} + +/// Compile the SQL relations and loader sources the selected checks depend on. +fn compile_pipeline<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, + (graph, check_names): (&Bound<'py, PyAny>, &Bound<'py, PyFrozenSet>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; + let required_refs: Bound<'py, PyAny> = call_with( + &graph.getattr("selected_sql_refs")?, + &[("selected_names", check_names.clone().into_any())], + )?; + let closure: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "transitive_closure_many")?, + &[ + ("starts", check_names.clone().into_any()), + ("edges", graph.getattr("upstream_deps")?), + ( + "include_starts", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?; + let loader_kind: Bound<'py, PyAny> = + imported(py, PYTHON_NODE_TYPES, "PythonNodeKind")?.getattr("LOADER")?; + let mut loaders: Vec> = Vec::new(); + for name in closure.try_iter()? { + let name: Bound<'py, PyAny> = name?; + if !check_names.contains(&name)? && node_kind(graph, &name)?.eq(&loader_kind)? { + loaders.push(name); + } + } + let loaders: Bound<'py, PyFrozenSet> = PyFrozenSet::new(py, loaders)?; + let key_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS, "CompiledObjectKey")?; + let source_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + let ref_key: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "sql_ref_key")?; + let mut keys: Vec> = Vec::new(); + for reference in required_refs.try_iter()? { + keys.push(ref_key.call1((reference?,))?); + } + for source_file in discovered.getattr("source_files")?.try_iter()? { + for source in source_file?.getattr("source_entries")?.try_iter()? { + let source: Bound<'py, PyAny> = source?; + if loaders.contains(source.getattr("loader")?)? { + keys.push(call_with( + &key_class, + &[ + ("resource_type", source_type.clone()), + ("name", source.getattr("name")?), + ], + )?); + } + } + } + let keys: Bound<'py, PyAny> = PyFrozenSet::new(py, keys)?.into_any(); + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.getattr("effective_project_dir")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let options: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &[ + ("selected_target", request.getattr("selected_target")?), + ("no_sql_validation", request.getattr("no_sql_validation")?), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("cli_vars", request.getattr("cli_vars")?), + ("external_sql_reference_resolver", resolver), + ], + )?; + let context: Bound<'py, PyAny> = call_with( + &imported(py, STATIC_COMMAND_MODULE, "compile_static_command_context")?, + &[ + ("discovered_inputs", discovered), + ("adapter", invocation.getattr("adapter")?), + ("options", options), + ("selected_keys", keys.clone()), + ("relation_keys", keys), + ( + "on_progress", + invocation + .getattr("planning_progress")? + .getattr("on_progress")?, + ), + ], + )?; + let plan_output: Bound<'py, PyAny> = call_with( + &imported(py, RELATION_PLAN_MODULE, "build_relation_command_plan")?, + &[ + ("project", context.getattr("project")?), + ("scope", context.getattr("scope")?), + ("relations", context.getattr("relations")?), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "build_static_pipeline_result")?, + &[("context", context), ("plan_output", plan_output)], + ) +} + +fn target_default<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + (field, fallback): (&str, &str), +) -> PyResult> { + let value: Bound<'py, PyAny> = project.getattr(field)?; + if value.is_none() { + adapter.call_method0(fallback) + } else { + Ok(value) + } +} + +/// The discovered Python graph, after refusing unit-test selectors, and the selected checks. +fn python_selection<'py>( + request: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyFrozenSet>)> { + let py: Python<'py> = request.py(); + let python_graph: Bound<'py, PyAny> = call_with( + &imported( + py, + PYTHON_GRAPH_MODULE, + "build_discovered_python_node_graph", + )?, + &[("discovered_inputs", discovered.clone())], + )?; + call_with( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let check_names: Bound<'py, PyFrozenSet> = selected_check_names(request, &python_graph)?; + Ok((python_graph, check_names)) +} + +fn prepare<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; + let adapter: Bound<'py, PyAny> = invocation.getattr("adapter")?; + let (planning_graph, planning_names) = python_selection(request, &discovered)?; + let pipeline: Bound<'py, PyAny> = + compile_pipeline(request, invocation, (&planning_graph, &planning_names))?; + let (python_graph, check_names) = python_selection(request, &discovered)?; + let mut check_functions: Vec> = Vec::new(); + for check in discovered.getattr("check_functions")?.try_iter()? { + let check: Bound<'py, PyAny> = check?; + if check_names.contains(check.getattr("name")?)? { + check_functions.push(check); + } + } + let selection: Bound<'py, PyAny> = call_with( + &imported(py, PYTHON_NODE_MODELS, "PythonSqlRunSelection")?, + &[ + ("sql_keys", PyFrozenSet::empty(py)?.into_any()), + ("python_node_names", PyFrozenSet::empty(py)?.into_any()), + ], + )?; + let lifecycle_plan: Bound<'py, PyAny> = call_with( + &imported( + py, + PYTHON_LIFECYCLE_MODULE, + "build_python_sql_run_lifecycle", + )?, + &[ + ("selection", selection), + ("python_graph", python_graph.clone()), + ], + )?; + let relation_refs: Bound<'py, PyAny> = call_with( + &python_graph.getattr("selected_sql_refs")?, + &[("selected_names", check_names.clone().into_any())], + )?; + let project: Bound<'py, PyAny> = pipeline.getattr("project")?; + let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let relation_targets: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_python_relation_targets")?, + &[ + ("adapter", adapter.clone()), + ("project", project.clone()), + ("plan_output", plan_output.clone()), + ("required_refs", relation_refs.clone()), + ], + )?; + Ok(Prepared { + check_functions: PyTuple::new(py, check_functions)?.into_any(), + lifecycle_plan, + relation_targets, + relation_refs, + defaults: ( + target_default( + &project, + &adapter, + ("effective_target_database", "default_database"), + )?, + target_default( + &project, + &adapter, + ("effective_target_schema", "default_schema"), + )?, + ), + project_relations: call_with( + &imported(py, NATIVE_MODULE, "build_project_relation_targets")?, + &[("adapter", adapter), ("plan_output", plan_output)], + )?, + python_graph, + pipeline, + }) +} + +/// The executor runtime of check dependencies and checks; ingress runs without project relations. +fn node_runtime<'py>( + invocation: &Bound<'py, PyAny>, + prepared: &Prepared<'py>, + (connection, providers, with_project_relations): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, bool), +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let project: Bound<'py, PyAny> = prepared.pipeline.getattr("project")?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("adapter", invocation.getattr("adapter")?), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("connection", connection.clone()), + ("run_id", project.getattr("run_id")?), + ("target", project.getattr("effective_target_name")?), + ("vars", project.getattr("effective_vars")?), + ("is_reload", PyBool::new(py, false).to_owned().into_any()), + ("default_database", prepared.defaults.0.clone()), + ("default_schema", prepared.defaults.1.clone()), + ("relation_targets", prepared.relation_targets.clone()), + ]; + if with_project_relations { + fields.push(("project_relations", prepared.project_relations.clone())); + } + fields.push(("providers", providers.clone())); + call_with( + &imported(py, EXECUTOR_MODELS, "PythonNodeRuntime")?, + &fields, + ) +} + +fn record_run_state<'py>( + invocation: &Bound<'py, PyAny>, + (run_state, python_results, load_results): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Option>, + ), + source_map: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = invocation.py(); + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ( + "discovered_inputs", + invocation.getattr("discovered_inputs")?, + ), + ("run_state", run_state), + ("python_results", python_results), + ]; + if let Some(load_results) = load_results { + fields.push(("load_results", load_results)); + } + fields.push(("source_map", source_map.clone())); + call_with( + &imported(py, CHECK_CORE_MODULE, "record_python_run_state_results")?, + &fields, + )?; + Ok(()) +} + +/// Run ingress loaders and read-side dependencies, then the checks themselves. +fn run_checks<'py>( + invocation: &Bound<'py, PyAny>, + prepared: &Prepared<'py>, + (connection, providers, event_writer): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let plan_output: Bound<'py, PyAny> = prepared.pipeline.getattr("plan_output")?; + let source_map: Bound<'py, PyAny> = plan_output.getattr("source_map")?; + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTOR_MODELS, "IngressCallbacks")?, + &[("use_color", invocation.getattr("use_color")?)], + )?; + let ingress: Bound<'py, PyAny> = call_with( + &imported(py, INGRESS_MODULE, "execute_ingress_python_loader_nodes")?, + &[ + ("python_graph", prepared.python_graph.clone()), + ( + "selected_python_names", + prepared + .lifecycle_plan + .getattr("ingress_python_node_names")?, + ), + ( + "loader_functions", + invocation + .getattr("discovered_inputs")? + .getattr("loader_functions")?, + ), + ("source_map", source_map.clone()), + ( + "runtime", + node_runtime(invocation, prepared, (connection, providers, false))?, + ), + ("callbacks", callbacks), + ], + )?; + let run_state: Bound<'py, PyAny> = ingress.getattr("run_state")?; + let load_results: Bound<'py, PyAny> = ingress.getattr("load_results")?; + let ingress_python: Bound<'py, PyAny> = ingress.getattr("python_results")?; + record_run_state( + invocation, + ( + run_state.clone(), + ingress_python.clone(), + Some(load_results.clone()), + ), + &source_map, + )?; + let read_side: Bound<'py, PyAny> = call_with( + &imported(py, CHECK_CORE_MODULE, "run_check_read_side_dependencies")?, + &[ + ("adapter", invocation.getattr("adapter")?), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("connection", connection.clone()), + ("pipeline_result", prepared.pipeline.clone()), + ("python_graph", prepared.python_graph.clone()), + ("lifecycle_plan", prepared.lifecycle_plan.clone()), + ("relation_targets", prepared.relation_targets.clone()), + ("validation_refs", prepared.relation_refs.clone()), + ("project_relations", prepared.project_relations.clone()), + ("providers", providers.clone()), + ], + )?; + record_run_state( + invocation, + (run_state.clone(), read_side.clone(), None), + &source_map, + )?; + let upstream: Vec> = ingress_python + .try_iter()? + .chain(read_side.try_iter()?) + .collect::>()?; + let by_loader: Bound<'py, PyAny> = call_with( + &imported(py, CHECK_CORE_MODULE, "load_results_by_loader_name")?, + &[ + ("source_map", source_map), + ("load_results", load_results.clone()), + ], + )?; + let on_complete: Bound<'py, PyAny> = Bound::new( + py, + CheckCompletionRelay { + event_writer: event_writer.clone().unbind(), + plan_output: plan_output.clone().unbind(), + }, + )? + .into_any(); + let check_callbacks: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTOR_MODELS, "PythonCheckCallbacks")?, + &[("on_check_complete", on_complete)], + )?; + call_with( + &imported(py, CHECKS_MODULE, "execute_python_checks")?, + &[ + ("check_functions", prepared.check_functions.clone()), + ("python_graph", prepared.python_graph.clone()), + ( + "upstream_python_results", + PyTuple::new(py, upstream)?.into_any(), + ), + ("upstream_load_results", load_results), + ("upstream_load_results_by_loader_name", by_loader), + ( + "runtime", + node_runtime(invocation, prepared, (connection, providers, true))?, + ), + ("run_state", run_state), + ( + "require_upstream_results", + PyBool::new(py, false).to_owned().into_any(), + ), + ("callbacks", check_callbacks), + ], + ) +} + +/// Record each finished check in the event output. +#[pyclass(module = "sqlbuild._native", frozen)] +struct CheckCompletionRelay { + event_writer: Py, + plan_output: Py, +} + +#[pymethods] +impl CheckCompletionRelay { + fn __call__(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("result", result)?; + options.set_item("plan", self.plan_output.bind(py))?; + options.set_item("command", "check")?; + self.event_writer + .bind(py) + .call_method("write_build_result", (), Some(&options))?; + Ok(()) + } +} + +/// Connect, configure progress ordinals, run the checks, and close everything. +fn execute<'py>( + invocation: &Bound<'py, PyAny>, + prepared: &Prepared<'py>, + providers: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = invocation.py(); + let adapter: Bound<'py, PyAny> = invocation.getattr("adapter")?; + let connection: Bound<'py, PyAny> = + adapter.call_method1("connect", (invocation.getattr("connection_config")?,))?; + let event_writer: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; + let executed: PyResult>> = (|| { + let projector: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + if !projector.is_none() { + let ordinals: Bound<'py, PyDict> = PyDict::new(py); + let mut total: usize = 0; + for check in prepared.check_functions.try_iter()? { + total += 1; + ordinals.set_item(check?.getattr("name")?, total)?; + } + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("ordinals", ordinals)?; + options.set_item("total", total)?; + projector.call_method("configure_resources", (), Some(&options))?; + } + run_checks( + invocation, + prepared, + (&connection, providers, &event_writer), + )? + .try_iter()? + .collect() + })(); + let writer_closed: PyResult> = event_writer.call_method0("close"); + let connection_closed: PyResult> = + adapter.call_method1("close", (&connection,)); + let results: Vec> = executed?; + writer_closed?; + connection_closed?; + Ok(results) +} + +/// Write rows, the summary, the runtime artifact and JSON output; returns the exit code. +fn complete<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, + (prepared, results): (&Prepared<'py>, &[Bound<'py, PyAny>]), +) -> PyResult { + let py: Python<'py> = request.py(); + let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; + let style: CliStyle = CliStyle { + use_color: invocation.getattr("use_color")?.extract()?, + }; + write_results( + &stream, + results, + (&prepared.check_functions, &prepared.python_graph), + style, + )?; + let texts: Vec = results + .iter() + .map(check_result_text) + .collect::>()?; + stream.call_method1("write", (render_check_summary(&texts, style),))?; + stream.call_method0("flush")?; + let tuple: Bound<'py, PyAny> = PyTuple::new(py, results)?.into_any(); + call_with( + &imported( + py, + RUNTIME_TARGET_MODULE, + "write_python_check_runtime_target", + )?, + &[ + ( + "target_dir", + invocation + .getattr("effective_project_dir")? + .call_method1("__truediv__", ("target",))?, + ), + ("results", tuple), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ( + "payload", + PyString::new(py, &results_json(results)?).into_any(), + ), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(check_exit_code(&texts)) +} + +/// Run one `sqb check` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let invocation: Bound<'_, PyAny> = call_with( + &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, + &[ + ("request", request.clone()), + ( + "invocation_type", + imported(py, COMMAND_MODELS, "CheckInvocation")?, + ), + ], + )?; + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + stream.call_method1("write", ("\n",))?; + let header: Bound<'_, PyDict> = PyDict::new(py); + header.set_item("stream", &stream)?; + header.set_item("command", EXECUTION_COMMAND)?; + header.set_item("target", py.None())?; + header.set_item("concurrency", PyInt::new(py, 1))?; + header.set_item("use_color", invocation.getattr("use_color")?)?; + imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&header))?; + let prepared: Prepared<'_> = prepare(request, &invocation)?; + let session: Bound<'_, PyAny> = call_with( + &imported(py, PROVIDER_SESSION_MODULE, "build_provider_session")?, + &[( + "discovered_providers", + invocation + .getattr("discovered_inputs")? + .getattr("providers")?, + )], + )?; + let outcome: PyResult = (|| { + let results: Vec> = + execute(&invocation, &prepared, &session.getattr("providers")?)?; + complete(request, &invocation, (&prepared, &results)) + })(); + let closed: PyResult> = session.call_method0("close"); + let code: i32 = outcome?; + closed?; + Ok(code) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_check_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs new file mode 100644 index 0000000000..92955c0b94 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs @@ -0,0 +1,238 @@ +//! Python check results as grouped rows and as the `sqb check --json` payload. + +use std::collections::HashMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::check_command::main::check_group_header::check_group_header; +use sqlbuild_core::check_command::main::check_group_label::check_group_label; +use sqlbuild_core::check_command::main::check_json::check_json; +use sqlbuild_core::check_command::main::render_check_row::render_check_row; +use sqlbuild_core::check_command::models::{CheckGrouping, CheckResultText, CheckRow, CheckStatus}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::json::models::JsonValue; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::result_output::integration_values::json_value; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const WARN_SEVERITY: &str = "warn"; + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +/// The metadata as JSON, through `json.dumps` when it holds values only that serializer accepts. +fn metadata_json(metadata: &Bound<'_, PyAny>) -> PyResult { + if let Some(value) = json_value(metadata)? { + return Ok(value); + } + let py: Python<'_> = metadata.py(); + let json: Bound<'_, PyAny> = py.import("json")?.into_any(); + let round_trip: Bound<'_, PyAny> = + json.call_method1("loads", (json.call_method1("dumps", (metadata,))?,))?; + json_value(&round_trip)? + .ok_or_else(|| PyValueError::new_err("check metadata is not JSON serializable")) +} + +/// One executor `PythonCheckExecutionResult` as the JSON output renders it, metadata included. +fn check_result_json_text(result: &Bound<'_, PyAny>) -> PyResult { + Ok(CheckResultText { + metadata: metadata_json(&result.getattr("metadata")?)?, + ..check_result_text(result)? + }) +} + +/// One executor `PythonCheckExecutionResult` as the rows and summary render it, without metadata. +pub(crate) fn check_result_text(result: &Bound<'_, PyAny>) -> PyResult { + let severity: String = result.getattr("severity")?.getattr("value")?.extract()?; + let status: CheckStatus = if result.getattr("passed")?.is_truthy()? { + CheckStatus::Pass + } else if severity == WARN_SEVERITY { + CheckStatus::Warn + } else { + CheckStatus::Fail + }; + Ok(CheckResultText { + name: result.getattr("node_name")?.extract()?, + status, + severity, + message: optional_text(result.getattr("message")?)?, + error_message: optional_text(result.getattr("error_message")?)?, + metadata: JsonValue::Null, + warning_messages: result.getattr("warning_messages")?.extract()?, + }) +} + +fn names(values: Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +/// The output group of `name`, from the Python graph and the check's declared group. +fn grouping( + name: &str, + checks: &HashMap>, + python_graph: &Bound<'_, PyAny>, +) -> PyResult { + let group: Option = match checks.get(name) { + Some(check) => optional_text(check.getattr("group")?)?, + None => None, + }; + if python_graph.is_none() { + return Ok(CheckGrouping { + dependencies: None, + group, + }); + } + let py: Python<'_> = python_graph.py(); + let empty: Bound<'_, PyAny> = pyo3::types::PyTuple::empty(py).into_any(); + let mut dependencies: Vec = names( + python_graph + .getattr("upstream_deps")? + .call_method1("get", (name, empty))?, + )?; + let node: Bound<'_, PyAny> = python_graph + .getattr("nodes_by_name")? + .call_method1("get", (name,))?; + if !node.is_none() { + for reference in node.getattr("sql_deps")?.try_iter()? { + dependencies.push(reference?.getattr("name")?.extract()?); + } + } + Ok(CheckGrouping { + dependencies: Some(dependencies), + group, + }) +} + +/// Write grouped check rows, claiming each row's projected duration once its row is written. +pub(crate) fn write_results( + stream: &Bound<'_, PyAny>, + results: &[Bound<'_, PyAny>], + (check_functions, python_graph): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + style: CliStyle, +) -> PyResult<()> { + let py: Python<'_> = stream.py(); + let mut checks: HashMap> = HashMap::new(); + for check in check_functions.try_iter()? { + let check: Bound<'_, PyAny> = check?; + checks.insert(check.getattr("name")?.extract()?, check); + } + let projector: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + let mut groups: Vec<(String, Vec)> = Vec::new(); + for result in results { + let text: CheckResultText = check_result_text(result)?; + let label: String = check_group_label(&grouping(&text.name, &checks, python_graph)?); + match groups.iter_mut().find(|(group, _)| *group == label) { + Some((_, members)) => members.push(text), + None => groups.push((label, vec![text])), + } + } + stream.call_method1("write", ("\n",))?; + for (group, members) in groups { + stream.call_method1("write", (check_group_header(&group),))?; + for result in members { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", &result.name)?; + options.set_item("resource_id", format!("check:{}", result.name))?; + let duration_ms: Option = if projector.is_none() { + None + } else { + projector + .call_method("resource_terminal_duration", (), Some(&options))? + .extract()? + }; + let row: CheckRow = CheckRow { + result, + duration_ms, + }; + stream.call_method1("write", (render_check_row(&row, style),))?; + if !projector.is_none() { + projector.call_method("consume_resource_terminal", (), Some(&options))?; + } + } + stream.call_method1("write", ("\n",))?; + } + Ok(()) +} + +/// The JSON payload of the results the terminal event index saw finish, or of all of them. +pub(crate) fn results_json(results: &[Bound<'_, PyAny>]) -> PyResult { + let Some(first) = results.first() else { + return check_json(&[]).map_err(|error| PyValueError::new_err(format!("{error:?}"))); + }; + let py: Python<'_> = first.py(); + let index: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "current_terminal_event_index")?.call0()?; + let mut texts: Vec = Vec::new(); + for result in results { + let name: Bound<'_, PyAny> = result.getattr("node_name")?; + if !index.is_none() { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", &name)?; + options.set_item( + "resource_id", + PyString::new(py, &format!("check:{}", name.str()?)), + )?; + if index + .call_method("resource_terminal", (), Some(&options))? + .is_none() + { + continue; + } + } + texts.push(check_result_json_text(result)?); + } + check_json(&texts).map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + +/// Write human-readable check result rows. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_check_results(py: Python<'_>, options: Option>) -> PyResult<()> { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "write_check_results", + options, + ( + &["stream", "results", "use_color"], + &["check_functions", "python_graph"], + ), + )?; + let results: Vec> = arguments + .value("results")? + .try_iter()? + .collect::>()?; + let check_functions: Bound<'_, PyAny> = match arguments.value("check_functions")? { + functions if functions.is_none() => pyo3::types::PyTuple::empty(py).into_any(), + functions => functions, + }; + write_results( + &arguments.value("stream")?, + &results, + (&check_functions, &arguments.value("python_graph")?), + CliStyle { + use_color: arguments.flag("use_color")?, + }, + ) +} + +/// Format check execution JSON. +#[pyfunction] +#[pyo3(signature = (*, results))] +fn format_check_json(results: &Bound<'_, PyAny>) -> PyResult { + let results: Vec> = results.try_iter()?.collect::>()?; + results_json(&results) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_check_results, module)?)?; + module.add_function(wrap_pyfunction!(format_check_json, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs new file mode 100644 index 0000000000..3bde938874 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod check_command; +pub(crate) mod check_results; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs new file mode 100644 index 0000000000..5baf04cec2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs @@ -0,0 +1,10 @@ +//! Register the native `sqb check` command and its result rendering. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::check::{check_command, check_results}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + check_command::register(module)?; + check_results::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 7c99056e02..032882f74a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -1,8 +1,10 @@ //! `sqb` command dispatch: the native route table applied to a parsed namespace. +use crate::bindings::_helpers::diff_report::support::cli_error; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_cli::diff_report::main::parse_diff_name_range::parse_diff_name_range; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; use sqlbuild_cli::dispatch::main::unavailable_handler::unavailable_handler; use sqlbuild_cli::dispatch::models::{ @@ -17,7 +19,6 @@ const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; const WAREHOUSES_MODULE: &str = "sqlbuild._native"; const WAREHOUSE_SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const LAUNCH_RECORD_MODULE: &str = "sqlbuild.cli.entry.main._withhold_launch_record"; -const DIFF_VALIDATION_MODULE: &str = "sqlbuild.cli.commands._helpers.diff.validation"; const FORMAT_PATHS_MODULE: &str = "sqlbuild.cli.commands._helpers.lint.positional_paths"; const PATHLIB_MODULE: &str = "pathlib"; const COMMENT_PREFIX: &str = "#"; @@ -121,13 +122,13 @@ impl<'py> Invocation<'py> { if range.is_none() { return Ok(range); } - let parsed: Bound<'py, PyAny> = - imported(self.py(), DIFF_VALIDATION_MODULE, "parse_diff_name_range")? - .call1((range,))?; - parsed.get_item(match side { - DiffSide::From => 0, - DiffSide::To => 1, - }) + let py: Python<'py> = self.py(); + let (from_name, to_name) = parse_diff_name_range(&range.extract::()?) + .map_err(|error| cli_error(py, "CliUserError", &error))?; + match side { + DiffSide::From => from_name.into_bound_py_any(py), + DiffSide::To => to_name.into_bound_py_any(py), + } } fn fields(&self, fields: &[Field]) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index eed0b86fe3..c947ab597f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -32,7 +32,6 @@ const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const CODED_ERROR_MODULE: &str = "sqlbuild.presentation.main.coded_error_text"; const TRANSIENT_LINE_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; const SKILLS_MODULE: &str = "sqlbuild._native"; -const DIFF_JSON_MODULE: &str = "sqlbuild.cli.commands._helpers.diff.json_output"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; @@ -195,7 +194,7 @@ fn render_diff_error<'py>( fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { call_with( - &imported(py, DIFF_JSON_MODULE, "render_diff_error_json")?, + &imported(py, NATIVE_MODULE, "render_diff_error_json")?, fields, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs index 50d1c0c5c7..a74d0e08c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs @@ -9,9 +9,10 @@ use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_core::progress::main::claim_progress_terminal::claim_progress_terminal; use sqlbuild_core::progress::main::close_progress_projection::close_progress_projection; use sqlbuild_core::progress::main::consume_progress_event::consume_progress_event; +use sqlbuild_core::progress::main::machine_output_mode::machine_output_mode as output_mode; use sqlbuild_core::progress::main::progress_terminal_duration::progress_terminal_duration; use sqlbuild_core::progress::models::{ - ProgressEvent, ProgressPayload, ProgressProjection, ProgressSettings, + OutputModeFacts, ProgressEvent, ProgressPayload, ProgressProjection, ProgressSettings, }; use sqlbuild_core::terminal_events::models::TerminalRequest; @@ -320,6 +321,22 @@ fn current_native_progress_projector(py: Python<'_>) -> PyResult) -> PyResult { + let mut dbt_args: Vec = Vec::new(); + for arg in args.getattr("dbt_args")?.try_iter()? { + dbt_args.push(arg?.str()?.extract()?); + } + Ok(output_mode(&OutputModeFacts { + json: args.getattr("json")?.is_truthy()?, + command: optional_text(args, "command")?, + lineage_format: optional_text(args, "lineage_format")?, + query_format: optional_text(args, "query_format")?, + dbt_args, + })) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { let py: Python<'_> = module.py(); let keywords: Bound<'_, PyDict> = PyDict::new(py); @@ -331,6 +348,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { .call((CONTEXT_NAME,), Some(&keywords))?, )?; module.add_class::()?; + module.add_function(wrap_pyfunction!(machine_output_mode, module)?)?; module.add_function(wrap_pyfunction!(current_native_progress_projector, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 5c21cfe7a3..179b9abaf4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -1,8 +1,13 @@ -//! Register the native `sqb` argument parsing and command dispatch. +//! Register the native `sqb` argument parsing, command dispatch and command handlers. use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; +use crate::bindings::_helpers::{ + check, clone, command_runtime, cost_report, dbt_interop, debug_command, diff_report, freshness, + janitor, load, presentation_text, result_output, seed_command, sql_test_command, + workspace_commands, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { argv::register(module)?; @@ -10,5 +15,20 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { progress::register(module)?; progress_reporters::register(module)?; progress_text::register(module)?; + janitor::registration::register(module)?; + freshness::registration::register(module)?; + clone::registration::register(module)?; + load::registration::register(module)?; + result_output::registration::register(module)?; + cost_report::registration::register(module)?; + debug_command::registration::register(module)?; + dbt_interop::registration::register(module)?; + diff_report::registration::register(module)?; + command_runtime::registration::register(module)?; + presentation_text::registration::register(module)?; + workspace_commands::registration::register(module)?; + check::registration::register(module)?; + sql_test_command::registration::register(module)?; + seed_command::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs new file mode 100644 index 0000000000..50470d396a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -0,0 +1,366 @@ +//! `sqb clone`: copy or clone selected relations from an origin target into a destination target. + +use std::collections::BTreeSet; +use std::time::Instant; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::main::format_phase_line::format_phase_line; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::clone_text::errors::CloneTargetError; +use sqlbuild_core::clone_text::main::clone_exit_code::clone_exit_code; +use sqlbuild_core::clone_text::main::render_clone_completion::render_clone_completion; +use sqlbuild_core::clone_text::main::render_clone_header::render_clone_header; +use sqlbuild_core::clone_text::main::validate_clone_policy::validate_clone_policy; +use sqlbuild_core::clone_text::main::validate_clone_target_names::validate_clone_target_names; +use sqlbuild_core::clone_text::models::CloneItemText; + +use crate::bindings::_helpers::clone::clone_execution::{ + destination_entry_count, execute_clone_plan, resource_types_by_name, +}; +use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const CLONE_PIPELINE_MODULE: &str = "sqlbuild.compiler.pipeline.main._clone"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const NO_DESTINATION: (&str, &str) = ( + "C406", + "clone requires --to when the project has no active target", +); +const NOTHING_TO_CLONE: (&str, &str) = + ("C407", "no cloneable resources found in the selected scope"); + +/// The discovered project, adapter and output context of one clone. +pub(crate) struct Invocation<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) discovered: Bound<'py, PyAny>, + pub(crate) adapter_name: String, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) destination: String, + pub(crate) style: CliStyle, + /// The stdout of the invocation, which progress keeps writing to while output is captured. + pub(crate) stream: Bound<'py, PyAny>, +} + +fn user_error(py: Python<'_>, (code, message): (&str, &str)) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", code)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn target_error(py: Python<'_>, error: CloneTargetError) -> PyErr { + user_error(py, (error.code, &error.message)) +} + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn progress_reporter<'py>( + stream: &Bound<'py, PyAny>, + style: CliStyle, +) -> PyResult> { + let py: Python<'py> = stream.py(); + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("stream", stream)?; + options.set_item("use_color", style.use_color)?; + imported(py, NATIVE_MODULE, "PlanningProgressReporter")?.call((), Some(&options)) +} + +fn lifecycle<'py>(py: Python<'py>, (kind, name): (&str, &str)) -> PyResult> { + call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", text(py, kind)), + ("operation_name", text(py, name)), + ], + ) +} + +fn target_names(discovered: &Bound<'_, PyAny>) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + for config in ["project_config", "local_config"] { + for name in discovered.getattr(config)?.getattr("targets")?.try_iter()? { + names.insert(name?.extract()?); + } + } + Ok(names) +} + +fn clone_policy(discovered: &Bound<'_, PyAny>, (target, flag): (&str, &str)) -> PyResult { + let py: Python<'_> = discovered.py(); + call_with( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("target_name", text(py, target)), + ], + )? + .getattr("clone")? + .getattr(flag)? + .is_truthy() +} + +/// Discover the project, resolve and validate the targets, and resolve the adapter. +fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult> { + let project_dir: Bound<'py, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, + }; + let use_color: bool = !request.getattr("no_color")?.extract::()? + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .extract::()?; + let style: CliStyle = CliStyle { use_color }; + let stream: Bound<'py, PyAny> = py.import("sys")?.getattr("stdout")?; + let progress: Bound<'py, PyAny> = progress_reporter(&stream, style)?; + progress.call_method1("start", ("Discovering project...",))?; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + progress.call_method1("complete", ("Discovered project.",))?; + call_with( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let mut destination: Bound<'py, PyAny> = request.getattr("destination_target_name")?; + if destination.is_none() { + destination = call_with( + &imported(py, TARGET_NAME_MODULE, "resolve_target_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("selected_target", py.None().into_bound(py)), + ], + )?; + } + if destination.is_none() { + return Err(user_error(py, NO_DESTINATION)); + } + let destination: String = destination.extract()?; + let origin: String = request.getattr("origin_target_name")?.extract()?; + let targets: (&str, &str) = (&origin, &destination); + validate_clone_target_names(targets, &target_names(&discovered)?) + .map_err(|error| target_error(py, error))?; + let policy: (bool, bool) = ( + clone_policy(&discovered, (&origin, "allow_as_clone_origin"))?, + clone_policy(&discovered, (&destination, "allow_as_clone_destination"))?, + ); + validate_clone_policy(targets, policy).map_err(|error| target_error(py, error))?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", project_dir.clone()), + ], + )?; + Ok(Invocation { + project_dir, + discovered, + adapter_name: adapter_name.extract()?, + adapter, + destination, + style, + stream, + }) +} + +/// Resolve the destination connection config and connect, returning both. +fn connect<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = request.py(); + let config: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_target_connection_config")?, + &[ + ("discovered_inputs", invocation.discovered.clone()), + ("project_dir", invocation.project_dir.clone()), + ("target_name", text(py, &invocation.destination)), + ("cli_vars", request.getattr("cli_vars")?), + ], + )?; + let progress: Bound<'py, PyAny> = progress_reporter(&invocation.stream, invocation.style)?; + progress.call_method1( + "start", + (format!("Connecting to {}...", invocation.adapter_name),), + )?; + let started: Instant = Instant::now(); + let connection: Bound<'py, PyAny> = within( + &lifecycle(py, ("clone", "clone_target_connection"))?, + |_| invocation.adapter.call_method1("connect", (&config,)), + )?; + let summary: String = format!( + "{} ({:.2}s)", + invocation.adapter_name, + started.elapsed().as_secs_f64() + ); + progress.call_method1( + "complete_styled", + (format_phase_line( + invocation.style, + true, + ("Warehouse connected", &summary), + ),), + )?; + Ok((config, connection)) +} + +/// Run the clone pipeline and refuse an empty selection. +fn prepare<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, + (config, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let progress: Bound<'py, PyAny> = progress_reporter(&invocation.stream, invocation.style)?; + let started: Instant = Instant::now(); + let pipeline: Bound<'py, PyAny> = + within(&lifecycle(py, ("project", "project_compile"))?, |_| { + progress.call_method1("on_progress", ("Preparing clone plan...",))?; + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.project_dir.clone()), + ("discovered_inputs", invocation.discovered.clone()), + ], + )?; + let destination_connection: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "ClonePipelineConnection")?, + &[("config", config.clone()), ("handle", connection.clone())], + )?; + let pipeline: Bound<'py, PyAny> = call_with( + &imported(py, CLONE_PIPELINE_MODULE, "run_clone_pipeline")?, + &[ + ("discovered_inputs", invocation.discovered.clone()), + ("adapter", invocation.adapter.clone()), + ("origin_target_name", request.getattr("origin_target_name")?), + ("destination_target_name", text(py, &invocation.destination)), + ("no_sql_validation", request.getattr("no_sql_validation")?), + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("cli_vars", request.getattr("cli_vars")?), + ("destination_connection", destination_connection), + ("external_sql_reference_resolver", resolver), + ], + )?; + progress.call_method1( + "on_progress", + (format!( + "Prepared clone plan. ({:.2}s)", + started.elapsed().as_secs_f64() + ),), + )?; + Ok(pipeline) + })?; + if destination_entry_count(&pipeline)? == 0 { + return Err(user_error(py, NOTHING_TO_CLONE)); + } + Ok(pipeline) +} + +/// Prepare, announce and execute the clone with an open destination connection. +fn clone<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, + (config, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>, f64)> { + let pipeline: Bound<'py, PyAny> = prepare(request, invocation, (config, connection))?; + let origin: String = request.getattr("origin_target_name")?.extract()?; + write_line( + &invocation.stream, + &render_clone_header( + (&origin, &invocation.destination), + destination_entry_count(&pipeline)?, + invocation.style, + ), + )?; + let (result, elapsed) = execute_clone_plan(request, invocation, (&pipeline, connection))?; + Ok((pipeline, result, elapsed)) +} + +/// Run one `sqb clone` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_clone_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let invocation: Invocation<'_> = invocation(py, request)?; + let (config, connection) = connect(request, &invocation)?; + let outcome = clone(request, &invocation, (&config, &connection)); + let closed: PyResult> = + invocation.adapter.call_method1("close", (&connection,)); + let (pipeline, result, elapsed) = outcome?; + closed?; + let items: Vec = result + .getattr("item_results")? + .try_iter()? + .map(|item| clone_item_text(&item?)) + .collect::>()?; + let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + for line in render_clone_completion(&items, Some(elapsed), invocation.style).lines() { + if !line.is_empty() { + stdout.call_method1("write", (line,))?; + } + stdout.call_method1("write", ("\n",))?; + } + let resource_types: Bound<'_, PyDict> = PyDict::new(py); + for (name, resource_type) in resource_types_by_name(&pipeline)? { + resource_types.set_item(name, resource_type)?; + } + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_clone_execution_json")?, + &[ + ("result", result.clone()), + ("resource_types_by_name", resource_types.into_any()), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", PyBool::new(py, false).to_owned().into_any()), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(clone_exit_code(&items)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_clone_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs new file mode 100644 index 0000000000..742ff72c25 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -0,0 +1,375 @@ +//! Execute the prepared clone plan and propagate fingerprints. + +use std::collections::HashMap; +use std::time::Instant; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString}; +use sqlbuild_core::clone_text::main::clone_relation_flow_text::clone_relation_flow_text; +use sqlbuild_core::clone_text::models::FingerprintProgress; + +use crate::bindings::_helpers::clone::clone_command::Invocation; +use crate::bindings::_helpers::clone::clone_progress::{ + CloneFingerprintReporter, CloneItemPrinter, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; +const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main.build_retention_requests"; +const TRANSIENT_MODELS_MODULE: &str = + "sqlbuild.executor.clone.main.build_destination_transient_models"; +const FINGERPRINTING_MODULE: &str = "sqlbuild.executor.clone.main.fingerprinting"; +const EXECUTE_MODULE: &str = "sqlbuild.executor.clone.main.execute"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ + "destination_source_entries", + "destination_model_entries", + "destination_seed_entries", + "destination_function_entries", +]; +const ORIGIN_ENTRY_GROUPS: [&str; 3] = [ + "origin_source_entries", + "origin_model_entries", + "origin_seed_entries", +]; + +/// The pipeline's plan entries of `groups`, in order. +fn entries<'py>(pipeline: &Bound<'py, PyAny>, groups: &[&str]) -> PyResult>> { + let mut found: Vec> = Vec::new(); + for group in groups { + for entry in pipeline.getattr(*group)?.try_iter()? { + found.push(entry?); + } + } + Ok(found) +} + +/// The number of destination entries, which the header counts. +pub(crate) fn destination_entry_count(pipeline: &Bound<'_, PyAny>) -> PyResult { + Ok(entries(pipeline, &DESTINATION_ENTRY_GROUPS)?.len()) +} + +/// Each destination entry's resource type by name. +pub(crate) fn resource_types_by_name( + pipeline: &Bound<'_, PyAny>, +) -> PyResult> { + entries(pipeline, &DESTINATION_ENTRY_GROUPS)? + .iter() + .map(|entry| { + Ok(( + entry.getattr("name")?.extract()?, + entry + .getattr("key")? + .getattr("resource_type")? + .str()? + .extract()?, + )) + }) + .collect() +} + +fn relation_text(entry: &Bound<'_, PyAny>) -> PyResult> { + let destination: Bound<'_, PyAny> = entry.getattr("destination")?; + let qualified: Bound<'_, PyAny> = destination.getattr("qualified_name")?; + let relation: Bound<'_, PyAny> = if qualified.is_truthy()? { + qualified + } else { + destination.getattr("name")? + }; + if relation.is_none() { + return Ok(None); + } + relation.extract().map(Some) +} + +/// The widest plain relation flow among the destination entries. +fn relation_width(pipeline: &Bound<'_, PyAny>) -> PyResult { + let mut origins: HashMap> = HashMap::new(); + for entry in entries(pipeline, &ORIGIN_ENTRY_GROUPS)? { + origins.insert(entry.getattr("name")?.extract()?, relation_text(&entry)?); + } + let mut width: usize = 0; + for entry in entries(pipeline, &DESTINATION_ENTRY_GROUPS)? { + let name: String = entry.getattr("name")?.extract()?; + let destination: Option = relation_text(&entry)?; + let origin: Option<&str> = origins.get(&name).and_then(Option::as_deref); + let flow: String = clone_relation_flow_text(&name, (origin, destination.as_deref())); + width = width.max(flow.chars().count()); + } + Ok(width) +} + +/// Destination relation locations of every planned entry and of each source table. +fn dependency_locations<'py>( + pipeline: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = pipeline.py(); + let locations: Bound<'py, PyDict> = PyDict::new(py); + for entry in entries(pipeline, &DESTINATION_ENTRY_GROUPS)? { + locations.set_item(entry.getattr("key")?, entry.getattr("destination")?)?; + } + let plan: Bound<'py, PyAny> = pipeline.getattr("clone_plan")?; + let read_map: Bound<'py, PyAny> = plan.getattr("source_read_map")?; + let source_map: Bound<'py, PyAny> = if read_map.is_truthy()? { + read_map + } else { + plan.getattr("source_map")? + }; + let key_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS, "CompiledObjectKey")?; + let location_class: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS, "CompiledRelationLocation")?; + let source_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + for pair in source_map.call_method0("items")?.try_iter()? { + let (name, source): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + let table: Bound<'py, PyAny> = source.getattr("table")?; + if table.is_none() { + continue; + } + let place = [ + ("database", source.getattr("database")?), + ("schema", source.getattr("schema")?), + ("name", table.clone()), + ]; + let rendered: Bound<'py, PyAny> = + call_with(&adapter.getattr("render_qualified_name")?, &place)?; + let qualified: Bound<'py, PyAny> = if rendered.is_truthy()? { + rendered + } else { + table + }; + let [database, schema, table] = place; + let location: Bound<'py, PyAny> = call_with( + &location_class, + &[database, schema, table, ("qualified_name", qualified)], + )?; + locations.set_item(key_class.call1((&source_type, name))?, location)?; + } + Ok(locations) +} + +fn target_config<'py>(invocation: &Invocation<'py>) -> PyResult> { + let py: Python<'py> = invocation.adapter.py(); + call_with( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ( + "project_config", + invocation.discovered.getattr("project_config")?, + ), + ( + "local_config", + invocation.discovered.getattr("local_config")?, + ), + ( + "target_name", + PyString::new(py, &invocation.destination).into_any(), + ), + ], + ) +} + +/// The executor input for the whole clone, streaming items through `on_item`. +fn execution_input<'py>( + invocation: &Invocation<'py>, + (pipeline, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (hard_copy, on_item): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = pipeline.py(); + let project: Bound<'py, PyAny> = pipeline.getattr("destination_project")?; + let plan: Bound<'py, PyAny> = pipeline.getattr("clone_plan")?; + let config: Bound<'py, PyAny> = target_config(invocation)?; + let selected: Vec> = pipeline + .getattr("destination_model_entries")? + .try_iter()? + .map(|entry| entry?.getattr("name")) + .collect::>()?; + let retention: Bound<'py, PyAny> = call_with( + &imported( + py, + RETENTION_REQUESTS_MODULE, + "build_destination_retention_requests", + )?, + &[ + ("project", project.clone()), + ( + "adapter_name", + PyString::new(py, &invocation.adapter_name).into_any(), + ), + ( + "selected_model_names", + PyFrozenSet::new(py, selected)?.into_any(), + ), + ( + "namespace_owned", + config.getattr("owns_time_travel_retention_namespace")?, + ), + ], + )?; + let allow_decrease: bool = config + .getattr("time_travel_retention_decrease")? + .eq(imported(py, CONTRACT_TYPES, "RetentionDecreasePolicy")?.getattr("ALLOW")?)?; + let source_entries: Bound<'py, PyAny> = call_with( + &imported(py, CLONE_MODELS, "CloneSourceEntries")?, + &[ + ("origin", pipeline.getattr("origin_source_entries")?), + ( + "destination", + pipeline.getattr("destination_source_entries")?, + ), + ], + )?; + let transient: Bound<'py, PyAny> = call_with( + &imported( + py, + TRANSIENT_MODELS_MODULE, + "build_destination_transient_models", + )?, + &[("project", project.clone())], + )?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![("source_entries", source_entries)]; + for group in [ + "origin_model_entries", + "destination_model_entries", + "origin_seed_entries", + "destination_seed_entries", + "destination_function_entries", + ] { + fields.push((group, pipeline.getattr(group)?)); + } + fields.extend([ + ("execution_order", plan.getattr("execution_order")?), + ("adapter", invocation.adapter.clone()), + ("destination_connection", connection.clone()), + ("hard_copy", hard_copy), + ("run_id", project.getattr("run_id")?), + ( + "query_change_tracking", + project + .getattr("settings")? + .getattr("query_change_tracking")?, + ), + ("upstream_deps", plan.getattr("upstream_deps")?), + ( + "dependency_locations", + dependency_locations(pipeline, &invocation.adapter)?.into_any(), + ), + ("destination_transient_models", transient), + ("destination_retention_requests", retention), + ( + "allow_namespace_retention_decrease", + PyBool::new(py, allow_decrease).to_owned().into_any(), + ), + ("on_item", on_item), + ]); + call_with(&imported(py, CLONE_MODELS, "CloneExecutionInput")?, &fields) +} + +/// Copy fingerprints of the cloned relations, reporting what was not confirmed on failure. +fn copy_fingerprints<'py>( + invocation: &Invocation<'py>, + (pipeline, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + result: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = pipeline.py(); + let reporter: Bound<'py, CloneFingerprintReporter> = Bound::new( + py, + CloneFingerprintReporter { + stream: invocation.stream.clone().unbind(), + style: invocation.style, + progress: FingerprintProgress::default(), + }, + )?; + let project: Bound<'py, PyAny> = pipeline.getattr("destination_project")?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![("result", result.clone())]; + for group in [ + "origin_model_entries", + "destination_model_entries", + "origin_seed_entries", + "destination_seed_entries", + ] { + fields.push((group, pipeline.getattr(group)?)); + } + fields.extend([ + ("adapter", invocation.adapter.clone()), + ("destination_connection", connection.clone()), + ("run_id", project.getattr("run_id")?), + ( + "query_change_tracking", + project + .getattr("settings")? + .getattr("query_change_tracking")?, + ), + ("on_progress", reporter.clone().into_any()), + ]); + let lifecycle: Bound<'py, PyAny> = call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", PyString::new(py, "clone").into_any()), + ( + "operation_name", + PyString::new(py, "clone_finalization").into_any(), + ), + ], + )?; + let copied: PyResult> = within(&lifecycle, |_| { + call_with( + &imported(py, FINGERPRINTING_MODULE, "copy_clone_fingerprints")?, + &fields, + ) + }); + if let Err(error) = copied { + reporter.borrow().write_interrupted(py)?; + return Err(error); + } + Ok(()) +} + +/// Execute the clone, streaming items, then copy fingerprints; returns the result and elapsed seconds. +pub(crate) fn execute_clone_plan<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, + (pipeline, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyAny>, f64)> { + let py: Python<'py> = pipeline.py(); + let started: Instant = Instant::now(); + let event_writer: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ExecutionEventWriter")?, + &[("path", request.getattr("event_output_path")?)], + )?; + let printer: Py = Py::new( + py, + CloneItemPrinter { + stream: invocation.stream.clone().unbind(), + style: invocation.style, + relation_width: relation_width(pipeline)?, + event_writer: event_writer.clone().unbind(), + resource_types: resource_types_by_name(pipeline)?, + }, + )?; + let executed: PyResult> = (|| { + let input: Bound<'py, PyAny> = execution_input( + invocation, + (pipeline, connection), + ( + request.getattr("hard_copy")?, + printer.into_bound(py).into_any(), + ), + )?; + call_with( + &imported(py, EXECUTE_MODULE, "execute_clone")?, + &[("inputs", input)], + ) + })(); + let closed: PyResult> = event_writer.call_method0("close"); + let result: Bound<'py, PyAny> = executed?; + closed?; + copy_fingerprints(invocation, (pipeline, connection), &result)?; + Ok((result, started.elapsed().as_secs_f64())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs new file mode 100644 index 0000000000..15a3ebb780 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs @@ -0,0 +1,129 @@ +//! Streamed clone item lines and fingerprint propagation progress, as executor callbacks. + +use std::collections::HashMap; + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::clone_text::main::render_clone_fingerprint_interrupted_line::render_clone_fingerprint_interrupted_line; +use sqlbuild_core::clone_text::main::render_clone_fingerprint_progress_line::render_clone_fingerprint_progress_line; +use sqlbuild_core::clone_text::main::render_clone_item_line::render_clone_item_line; +use sqlbuild_core::clone_text::models::{CloneItemText, CloneStatusKind, FingerprintProgress}; + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +/// One executor `CloneItemResult` as the command renders it. +pub(crate) fn clone_item_text(item: &Bound<'_, PyAny>) -> PyResult { + let duration: Bound<'_, PyAny> = item.getattr("duration_seconds")?; + Ok(CloneItemText { + name: item.getattr("name")?.extract()?, + action: item.getattr("action")?.getattr("value")?.extract()?, + status: CloneStatusKind::from_value( + &item + .getattr("status")? + .getattr("value")? + .extract::()?, + ), + message: optional_text(item.getattr("message")?)?, + origin_relation: optional_text(item.getattr("origin_relation")?)?, + destination_relation: optional_text(item.getattr("destination_relation")?)?, + duration_seconds: if duration.is_none() { + None + } else { + Some(duration.extract()?) + }, + }) +} + +/// Write `line` and a newline to `stream` and flush. +pub(crate) fn write_line(stream: &Bound<'_, PyAny>, line: &str) -> PyResult<()> { + stream.call_method1("write", (format!("{line}\n"),))?; + stream.call_method0("flush")?; + Ok(()) +} + +/// Print each finished clone item and record it in the event output. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CloneItemPrinter { + pub(crate) stream: Py, + pub(crate) style: CliStyle, + pub(crate) relation_width: usize, + pub(crate) event_writer: Py, + pub(crate) resource_types: HashMap, +} + +#[pymethods] +impl CloneItemPrinter { + #[pyo3(signature = (*, index, total, item))] + fn __call__( + &self, + py: Python<'_>, + index: usize, + total: usize, + item: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let text: CloneItemText = clone_item_text(item)?; + write_line( + self.stream.bind(py), + &render_clone_item_line((index, total), &text, self.relation_width, self.style), + )?; + let resource_type: &String = self + .resource_types + .get(&text.name) + .ok_or_else(|| PyKeyError::new_err(text.name.clone()))?; + let options: Bound<'_, PyDict> = PyDict::new(py); + pyo3::types::PyDictMethods::set_item(&options, "item", item)?; + pyo3::types::PyDictMethods::set_item(&options, "resource_type", resource_type)?; + self.event_writer + .bind(py) + .call_method("write_clone_result", (), Some(&options))?; + Ok(()) + } +} + +/// Report confirmed fingerprint batches and, on interruption, what was not confirmed. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct CloneFingerprintReporter { + pub(crate) stream: Py, + pub(crate) style: CliStyle, + pub(crate) progress: FingerprintProgress, +} + +#[pymethods] +impl CloneFingerprintReporter { + #[pyo3(signature = (*, completed, total, pending_identities))] + fn __call__( + &mut self, + py: Python<'_>, + completed: i64, + total: i64, + pending_identities: Vec, + ) -> PyResult<()> { + self.progress = FingerprintProgress { + completed, + total, + pending_identities, + }; + if completed == 0 { + return Ok(()); + } + write_line( + self.stream.bind(py), + &render_clone_fingerprint_progress_line(completed, total, self.style), + ) + } + + pub(crate) fn write_interrupted(&self, py: Python<'_>) -> PyResult<()> { + match render_clone_fingerprint_interrupted_line(&self.progress, self.style) { + Some(line) => write_line(self.stream.bind(py), &line), + None => Ok(()), + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs new file mode 100644 index 0000000000..6ac5b67d26 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod clone_command; +pub(crate) mod clone_execution; +pub(crate) mod clone_progress; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs new file mode 100644 index 0000000000..9df257097e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs @@ -0,0 +1,9 @@ +//! Register the native `sqb clone` command. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::clone::clone_command; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + clone_command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index fe3b51108a..be33723125 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -29,7 +29,7 @@ const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_na const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main.session_warehouse_support"; const DISCOVER_CONFIGURATION_MODULE: &str = - "sqlbuild.compiler.discovery.main.discover_configuration"; + "sqlbuild.compiler.discovery.main._discover_configuration"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild.compiler.compile.main.expand_template_data"; const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs new file mode 100644 index 0000000000..31f7a842d3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs @@ -0,0 +1,594 @@ +//! dbt manifest loading, indexing and `__dbt_ref` resolution. + +use std::collections::HashSet; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::dbt_manifest::constants::{ + MODEL_RESOURCE_TYPE, NODE_TEXT_FIELDS, SEED_RESOURCE_TYPE, SNAPSHOT_RESOURCE_TYPE, + SOURCE_RESOURCE_TYPE, TEST_RESOURCE_TYPE, UNIT_TEST_RESOURCE_TYPE, +}; +use sqlbuild_config::dbt_manifest::errors::ManifestError; +use sqlbuild_config::dbt_manifest::main::parse_manifest_model::parse_manifest_model; +use sqlbuild_config::dbt_manifest::main::parse_manifest_seed::parse_manifest_seed; +use sqlbuild_config::dbt_manifest::main::parse_manifest_source::parse_manifest_source; +use sqlbuild_config::dbt_manifest::models::{ + ManifestNode, NodeChecksum, ParsedModel, ParsedSeed, ParsedSource, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const DBT_REF_KIND: &str = "dbt_ref"; +const RELATION_OPTIONS: (&[&str], &[&str]) = + (&["manifest", "ref_kind", "ref_name", "ref_package"], &[]); + +fn raised(py: Python<'_>, error: ManifestError) -> PyErr { + let created = || -> PyResult { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", error.code)?; + keywords.set_item("help", error.help)?; + let instance: Bound<'_, PyAny> = imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")? + .call((error.message,), Some(&keywords))?; + Ok(PyErr::from_value(instance)) + }; + created().unwrap_or_else(|failure| failure) +} + +fn text_value(value: Option>) -> PyResult> { + match value { + Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), + _ => Ok(None), + } +} + +fn checksum(value: Option>) -> PyResult { + let Some(value) = value else { + return Ok(NodeChecksum::Other); + }; + if value.is_instance_of::() { + return Ok(NodeChecksum::Text(value.extract()?)); + } + match value.cast::() { + Ok(nested) => Ok(NodeChecksum::Nested(text_value( + nested.get_item("checksum")?, + )?)), + Err(_) => Ok(NodeChecksum::Other), + } +} + +fn fqn(value: Option>) -> PyResult>>> { + let Some(value) = value.filter(|value| value.is_instance_of::()) else { + return Ok(None); + }; + let mut parts: Vec> = Vec::new(); + for part in value.try_iter()? { + parts.push(text_value(Some(part?))?); + } + Ok(Some(parts)) +} + +fn depends_on(value: Option>) -> PyResult> { + let Some(Ok(depends_on)) = value.map(|value| value.cast_into::()) else { + return Ok(Vec::new()); + }; + let Some(nodes) = depends_on + .get_item("nodes")? + .filter(|nodes| nodes.is_instance_of::()) + else { + return Ok(Vec::new()); + }; + let mut unique_ids: Vec = Vec::new(); + for node in nodes.try_iter()? { + if let Some(unique_id) = text_value(Some(node?))? { + unique_ids.push(unique_id); + } + } + Ok(unique_ids) +} + +fn manifest_node(raw: &Bound<'_, PyDict>) -> PyResult { + let mut text: Vec<(String, String)> = Vec::new(); + for field in NODE_TEXT_FIELDS { + if let Some(value) = text_value(raw.get_item(field)?)? { + text.push((field.to_owned(), value)); + } + } + Ok(ManifestNode { + text, + checksum: checksum(raw.get_item("checksum")?)?, + fqn: fqn(raw.get_item("fqn")?)?, + depends_on: depends_on(raw.get_item("depends_on")?)?, + }) +} + +fn resource_type(raw: &Bound<'_, PyDict>) -> PyResult> { + text_value(raw.get_item("resource_type")?) +} + +/// `{str(key): value for key, value in raw_node.items()}`. +fn payload<'py>(raw: &Bound<'py, PyDict>) -> PyResult> { + let payload: Bound<'py, PyDict> = PyDict::new(raw.py()); + for (key, value) in raw.iter() { + payload.set_item(key.str()?, value)?; + } + Ok(payload.into_any()) +} + +fn optional<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { + match value { + Some(text) => PyString::new(py, text).into_any(), + None => py.None().into_bound(py), + } +} + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn model_object<'py>( + unique_id: &Bound<'py, PyAny>, + model: &ParsedModel, + raw: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = raw.py(); + call_with( + &imported(py, DBT_MODELS, "DbtManifestModel")?, + &[ + ("unique_id", unique_id.clone()), + ("package_name", text(py, &model.package_name)), + ("name", text(py, &model.name)), + ("database", optional(py, model.database.as_deref())), + ("schema", optional(py, model.schema.as_deref())), + ("alias", optional(py, model.alias.as_deref())), + ( + "node_checksum", + optional(py, model.node_checksum.as_deref()), + ), + ("relation_name", text(py, &model.relation_name)), + ("fqn", PyTuple::new(py, &model.fqn)?.into_any()), + ("query_sql", text(py, &model.query_sql)), + ( + "depends_on_nodes", + PyTuple::new(py, &model.depends_on)?.into_any(), + ), + ("payload", payload(raw)?), + ], + ) +} + +fn seed_object<'py>( + unique_id: &Bound<'py, PyAny>, + seed: &ParsedSeed, + raw: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = raw.py(); + call_with( + &imported(py, DBT_MODELS, "DbtManifestSeed")?, + &[ + ("unique_id", unique_id.clone()), + ("package_name", text(py, &seed.package_name)), + ("name", text(py, &seed.name)), + ("database", optional(py, seed.database.as_deref())), + ("schema", optional(py, seed.schema.as_deref())), + ("alias", optional(py, seed.alias.as_deref())), + ("relation_name", text(py, &seed.relation_name)), + ("payload", payload(raw)?), + ], + ) +} + +fn source_object<'py>( + unique_id: &Bound<'py, PyAny>, + source: &ParsedSource, + raw: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = raw.py(); + call_with( + &imported(py, DBT_MODELS, "DbtManifestSource")?, + &[ + ("unique_id", unique_id.clone()), + ("package_name", text(py, &source.package_name)), + ("source_name", text(py, &source.source_name)), + ("name", text(py, &source.name)), + ("database", optional(py, source.database.as_deref())), + ("schema", optional(py, source.schema.as_deref())), + ("identifier", optional(py, source.identifier.as_deref())), + ("relation_name", text(py, &source.relation_name)), + ("payload", payload(raw)?), + ], + ) +} + +/// The manifest section `key`: an object, or empty when missing and `optional`. +fn section<'py>( + manifest: &Bound<'py, PyDict>, + (key, optional_section): (&str, bool), +) -> PyResult> { + let py: Python<'py> = manifest.py(); + match manifest.get_item(key)? { + None if optional_section => Ok(PyDict::new(py)), + Some(value) if value.is_none() && optional_section => Ok(PyDict::new(py)), + Some(value) => value.cast_into::().map_err(|_| { + raised( + py, + ManifestError::invalid(&format!(": {key} must be an object")), + ) + }), + None => Err(raised( + py, + ManifestError::invalid(&format!(": {key} must be an object")), + )), + } +} + +/// `(unique_id, node)` pairs whose key is a string and value an object. +fn node_items<'py>(nodes: &Bound<'py, PyDict>) -> Vec<(Bound<'py, PyAny>, Bound<'py, PyDict>)> { + let mut items: Vec<(Bound<'py, PyAny>, Bound<'py, PyDict>)> = Vec::new(); + for (key, value) in nodes.iter() { + if !key.is_instance_of::() { + continue; + } + if let Ok(node) = value.cast_into::() { + items.push((key, node)); + } + } + items +} + +/// The model indexes built from `nodes`, in manifest order. +struct ModelIndexes<'py> { + by_unique_id: Bound<'py, PyDict>, + by_name: Bound<'py, PyDict>, + by_package_and_name: Bound<'py, PyDict>, + seeds: Bound<'py, PyDict>, +} + +fn index_nodes<'py>(nodes: &Bound<'py, PyDict>) -> PyResult> { + let py: Python<'py> = nodes.py(); + let indexes: ModelIndexes<'py> = ModelIndexes { + by_unique_id: PyDict::new(py), + by_name: PyDict::new(py), + by_package_and_name: PyDict::new(py), + seeds: PyDict::new(py), + }; + for (unique_id, raw) in node_items(nodes) { + let kind: Option = resource_type(&raw)?; + let kind: &str = kind.as_deref().unwrap_or_default(); + if kind == MODEL_RESOURCE_TYPE || kind == SNAPSHOT_RESOURCE_TYPE { + let model: ParsedModel = + parse_manifest_model(&manifest_node(&raw)?).map_err(|error| raised(py, error))?; + let object: Bound<'py, PyAny> = model_object(&unique_id, &model, &raw)?; + indexes.by_unique_id.set_item(&unique_id, &object)?; + let named: Bound<'py, PyAny> = match indexes.by_name.get_item(&model.name)? { + Some(named) => named, + None => { + let named: Bound<'py, PyList> = PyList::empty(py); + indexes.by_name.set_item(&model.name, &named)?; + named.into_any() + } + }; + named.call_method1("append", (&object,))?; + indexes + .by_package_and_name + .set_item((&model.package_name, &model.name), &object)?; + } else if kind == SEED_RESOURCE_TYPE { + let seed: ParsedSeed = + parse_manifest_seed(&manifest_node(&raw)?).map_err(|error| raised(py, error))?; + indexes + .seeds + .set_item(&unique_id, seed_object(&unique_id, &seed, &raw)?)?; + } + } + let named: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = indexes.by_name.iter().collect(); + for (name, models) in named { + let models: Vec> = models.try_iter()?.collect::>()?; + indexes.by_name.set_item(name, PyTuple::new(py, models)?)?; + } + Ok(indexes) +} + +fn validation_dependencies<'py>( + sections: [&Bound<'py, PyDict>; 2], +) -> PyResult> { + let py: Python<'py> = sections[0].py(); + let dependencies: Bound<'py, PyDict> = PyDict::new(py); + for nodes in sections { + for (unique_id, raw) in node_items(nodes) { + let kind: Option = resource_type(&raw)?; + if kind.as_deref() != Some(TEST_RESOURCE_TYPE) + && kind.as_deref() != Some(UNIT_TEST_RESOURCE_TYPE) + { + continue; + } + let unique_ids: Vec = depends_on(raw.get_item("depends_on")?)? + .into_iter() + .filter(|node| !node.is_empty()) + .collect(); + dependencies.set_item(unique_id, PyTuple::new(py, unique_ids)?)?; + } + } + Ok(dependencies) +} + +fn manifest_index<'py>(raw_data: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = raw_data.py(); + let manifest: &Bound<'py, PyDict> = raw_data + .cast::() + .map_err(|_| raised(py, ManifestError::invalid(": root must be an object")))?; + let nodes: Bound<'py, PyDict> = section(manifest, ("nodes", false))?; + let sources: Bound<'py, PyDict> = section(manifest, ("sources", true))?; + let unit_tests: Bound<'py, PyDict> = section(manifest, ("unit_tests", true))?; + let indexes: ModelIndexes<'py> = index_nodes(&nodes)?; + let validation: Bound<'py, PyDict> = validation_dependencies([&nodes, &unit_tests])?; + let sources_by_unique_id: Bound<'py, PyDict> = PyDict::new(py); + for (unique_id, raw) in node_items(&sources) { + if resource_type(&raw)?.as_deref() != Some(SOURCE_RESOURCE_TYPE) { + continue; + } + let source: ParsedSource = + parse_manifest_source(&manifest_node(&raw)?).map_err(|error| raised(py, error))?; + sources_by_unique_id.set_item(&unique_id, source_object(&unique_id, &source, &raw)?)?; + } + call_with( + &imported(py, DBT_MODELS, "DbtManifestIndex")?, + &[ + ("models_by_unique_id", indexes.by_unique_id.into_any()), + ("models_by_name", indexes.by_name.into_any()), + ( + "models_by_package_and_name", + indexes.by_package_and_name.into_any(), + ), + ("sources_by_unique_id", sources_by_unique_id.into_any()), + ("seeds_by_unique_id", indexes.seeds.into_any()), + ( + "validation_depends_on_nodes_by_unique_id", + validation.into_any(), + ), + ], + ) +} + +/// `json.loads(contents)`, with a decode failure raised as an invalid manifest. +fn decoded<'py>(contents: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = contents.py(); + match imported(py, "json", "loads")?.call1((contents,)) { + Ok(value) => Ok(value), + Err(error) if error.is_instance(py, &imported(py, "json", "JSONDecodeError")?) => { + let detail: String = error.value(py).getattr("msg")?.extract()?; + Err(raised( + py, + ManifestError::invalid(&format!(" JSON: {detail}")), + )) + } + Err(error) => Err(error), + } +} + +/// Load and index dbt model nodes from a manifest.json file. +#[pyfunction] +#[pyo3(signature = (*, manifest_path))] +fn load_dbt_manifest_index<'py>( + py: Python<'py>, + manifest_path: Bound<'py, PyAny>, +) -> PyResult> { + if !manifest_path.call_method0("is_file")?.is_truthy()? { + let path: String = manifest_path.str()?.extract()?; + return Err(raised(py, ManifestError::missing_manifest_file(&path))); + } + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("encoding", "utf-8")?; + let contents: Bound<'py, PyAny> = + manifest_path.call_method("read_text", (), Some(&keywords))?; + manifest_index(&decoded(&contents)?) +} + +/// Build a dbt manifest index from discovered manifest contents. +#[pyfunction] +#[pyo3(signature = (*, manifest_contents))] +fn build_compile_dbt_manifest_index<'py>( + py: Python<'py>, + manifest_contents: Bound<'py, PyAny>, +) -> PyResult> { + if manifest_contents.is_none() { + return Ok(py.None().into_bound(py)); + } + manifest_index(&decoded(&manifest_contents)?) +} + +/// Build dbt model lookup indexes from decoded manifest JSON. +#[pyfunction] +#[pyo3(signature = (*, raw_data))] +fn build_dbt_manifest_index<'py>(raw_data: Bound<'py, PyAny>) -> PyResult> { + manifest_index(&raw_data) +} + +/// The manifest model a one-arg or package-qualified `__dbt_ref` names. +fn resolved_model<'py>( + manifest: &Bound<'py, PyAny>, + name: &str, + package_name: Option<&str>, +) -> PyResult> { + let py: Python<'py> = manifest.py(); + if let Some(package) = package_name { + let model: Bound<'py, PyAny> = manifest + .getattr("models_by_package_and_name")? + .call_method1("get", ((package, name),))?; + if model.is_none() { + return Err(raised( + py, + ManifestError::unknown_model(name, Some(package)), + )); + } + return Ok(model); + } + let matches: Bound<'py, PyAny> = manifest + .getattr("models_by_name")? + .call_method1("get", (name, PyTuple::empty(py)))?; + let matches: Vec> = matches.try_iter()?.collect::>()?; + match matches.as_slice() { + [] => Err(raised(py, ManifestError::unknown_model(name, None))), + [model] => Ok(model.clone()), + _ => { + let packages: Vec = matches + .iter() + .map(|model| model.getattr("package_name")?.extract()) + .collect::>()?; + Err(raised(py, ManifestError::ambiguous_model(name, packages))) + } + } +} + +/// Resolve a dbt model by one-arg or package-qualified ref semantics. +#[pyfunction] +#[pyo3(signature = (*, manifest, name, package_name = None))] +fn resolve_dbt_manifest_model<'py>( + manifest: Bound<'py, PyAny>, + name: String, + package_name: Option, +) -> PyResult> { + resolved_model(&manifest, &name, package_name.as_deref()) +} + +/// Reject ambiguous ownership between dbt and SQLBuild models. +#[pyfunction] +#[pyo3(signature = (*, known_model_names, dbt_manifest))] +fn validate_compile_dbt_model_names( + py: Python<'_>, + known_model_names: Bound<'_, PyAny>, + dbt_manifest: Bound<'_, PyAny>, +) -> Result<(), PyErr> { + if dbt_manifest.is_none() { + return Ok(()); + } + let dbt_names: Bound<'_, PyAny> = dbt_manifest.getattr("models_by_name")?; + let mut shared: Vec = Vec::new(); + for name in known_model_names.try_iter()? { + let name: Bound<'_, PyAny> = name?; + if dbt_names.contains(&name)? { + shared.push(name.extract()?); + } + } + match ManifestError::shared_model_names(shared) { + Some(error) => Err(raised(py, error)), + None => Ok(()), + } +} + +fn dbt_ref_kind(ref_kind: &Bound<'_, PyAny>) -> PyResult { + ref_kind.eq(DBT_REF_KIND) +} + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +/// Validate one compile-time dbt ref against a dbt manifest. +#[pyfunction] +#[pyo3(signature = (*, reference, model_relative_path, dbt_manifest))] +fn validate_compile_dbt_model_reference( + py: Python<'_>, + reference: Bound<'_, PyAny>, + model_relative_path: Bound<'_, PyAny>, + dbt_manifest: Bound<'_, PyAny>, +) -> Result<(), PyErr> { + if !dbt_ref_kind(&reference.getattr("ref_kind")?)? { + return Ok(()); + } + let name: String = reference.getattr("ref_name")?.extract()?; + if dbt_manifest.is_none() { + let path: String = model_relative_path.str()?.extract()?; + return Err(raised(py, ManifestError::missing_manifest(&path, &name))); + } + let package: Option = optional_text(&reference.getattr("ref_package")?)?; + resolved_model(&dbt_manifest, &name, package.as_deref())?; + Ok(()) +} + +fn name_set(names: &Bound<'_, PyAny>) -> PyResult> { + let mut set: HashSet = HashSet::new(); + for name in names.try_iter()? { + set.insert(name?.extract()?); + } + Ok(set) +} + +/// Return resolved dbt refs used directly by SQLBuild models. +#[pyfunction] +#[pyo3(signature = (*, project, manifest, selected_model_names = None))] +fn resolve_sqlbuild_model_dbt_refs<'py>( + py: Python<'py>, + project: Bound<'py, PyAny>, + manifest: Bound<'py, PyAny>, + selected_model_names: Option>, +) -> PyResult> { + let selected_model_names: Option> = selected_model_names + .map(|names| name_set(&names)) + .transpose()?; + let mut refs: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + if let Some(selected) = &selected_model_names { + let name: String = model.getattr("name")?.extract()?; + if !selected.contains(&name) { + continue; + } + } + for reference in model.getattr("references")?.try_iter()? { + let reference: Bound<'py, PyAny> = reference?; + if !dbt_ref_kind(&reference.getattr("ref_kind")?)? { + continue; + } + let name: String = reference.getattr("ref_name")?.extract()?; + let package: Option = optional_text(&reference.getattr("ref_package")?)?; + let resolved: Bound<'py, PyAny> = resolved_model(&manifest, &name, package.as_deref())?; + refs.push(PyTuple::new(py, [model.clone(), resolved])?); + } + } + PyTuple::new(py, refs) +} + +/// Resolve one external dbt reference to its manifest relation. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_dbt_reference_relation<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_dbt_reference_relation", + options, + RELATION_OPTIONS, + )?; + let manifest: Bound<'py, PyAny> = arguments.value("manifest")?; + if !dbt_ref_kind(&arguments.value("ref_kind")?)? || manifest.is_none() { + return Ok(py.None().into_bound(py)); + } + let name: String = arguments.value("ref_name")?.extract()?; + let package: Option = optional_text(&arguments.value("ref_package")?)?; + resolved_model(&manifest, &name, package.as_deref())?.getattr("relation_name") +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(load_dbt_manifest_index, module)?)?; + module.add_function(wrap_pyfunction!(build_compile_dbt_manifest_index, module)?)?; + module.add_function(wrap_pyfunction!(build_dbt_manifest_index, module)?)?; + module.add_function(wrap_pyfunction!(resolve_dbt_manifest_model, module)?)?; + module.add_function(wrap_pyfunction!(validate_compile_dbt_model_names, module)?)?; + module.add_function(wrap_pyfunction!( + validate_compile_dbt_model_reference, + module + )?)?; + module.add_function(wrap_pyfunction!(resolve_sqlbuild_model_dbt_refs, module)?)?; + module.add_function(wrap_pyfunction!(resolve_dbt_reference_relation, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs new file mode 100644 index 0000000000..40ad34c6e3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod manifest; +pub(crate) mod plan; +pub(crate) mod profile; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs new file mode 100644 index 0000000000..82c1cbc183 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs @@ -0,0 +1,254 @@ +//! Build and render the `sqb dbt` selection plan. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::main::dbt_interop_plan_warnings::dbt_interop_plan_warnings; +use sqlbuild_core::cli_text::main::format_dbt_interop_plan::format_dbt_interop_plan; +use sqlbuild_core::cli_text::main::format_dbt_interop_plan_json::format_dbt_interop_plan_json; +use sqlbuild_core::cli_text::models::{DbtInteropPlanText, DbtLsNodeText, DisplayLimit}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const DBT_TYPES: &str = "sqlbuild.integrations.dbt.types"; +const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; +const BUILD_OPTIONS: (&[&str], &[&str]) = ( + &[ + "command", + "dbt_command_argv", + "dbt_ls_nodes", + "sqlbuild_command_argvs", + "selection", + ], + &[ + "dbt_required_selector_terms", + "supplemental_dbt_command_argvs", + "warnings", + "sqlbuild_plan_output", + ], +); +const RENDER_OPTIONS: (&[&str], &[&str]) = + (&["plan", "json_output", "use_color"], &["display_options"]); + +fn strings(value: &Bound<'_, PyAny>) -> PyResult> { + value.try_iter()?.map(|item| item?.extract()).collect() +} + +fn argvs(value: &Bound<'_, PyAny>) -> PyResult>> { + value.try_iter()?.map(|argv| strings(&argv?)).collect() +} + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +fn reason_value(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.getattr("value")?.extract()?)) +} + +fn ls_node(node: &Bound<'_, PyAny>) -> PyResult { + Ok(DbtLsNodeText { + unique_id: node.getattr("unique_id")?.extract()?, + resource_type: optional_text(&node.getattr("resource_type")?)?, + name: optional_text(&node.getattr("name")?)?, + fqn: strings(&node.getattr("fqn")?)?, + }) +} + +fn plan_text(plan: &Bound<'_, PyAny>) -> PyResult { + let selection: Bound<'_, PyAny> = plan.getattr("selection")?; + let anchor_ids: Bound<'_, PyAny> = selection.getattr("dbt_anchor_unique_ids_by_term")?; + let empty: Bound<'_, PyTuple> = PyTuple::empty(plan.py()); + let mut anchors: Vec<(String, Vec)> = Vec::new(); + for term in selection.getattr("dbt_anchor_terms")?.try_iter()? { + let term: Bound<'_, PyAny> = term?; + let unique_ids: Vec = strings(&anchor_ids.call_method1("get", (&term, &empty))?)?; + anchors.push((term.extract()?, unique_ids)); + } + let mut path_translations: Vec<(String, String)> = Vec::new(); + for pair in selection.getattr("path_translations")?.try_iter()? { + path_translations.push(pair?.extract()?); + } + Ok(DbtInteropPlanText { + command: plan.getattr("command")?.getattr("value")?.extract()?, + dbt_argv: strings(&plan.getattr("dbt_command_argv")?)?, + supplemental_dbt_argvs: argvs(&plan.getattr("supplemental_dbt_command_argvs")?)?, + dbt_nodes: plan + .getattr("dbt_selected_nodes")? + .try_iter()? + .map(|node| ls_node(&node?)) + .collect::>()?, + dbt_selected_unique_ids: strings(&plan.getattr("dbt_selected_unique_ids")?)?, + dbt_required_unique_ids: strings(&selection.getattr("dbt_required_unique_ids")?)?, + dbt_required_selector_terms: strings(&plan.getattr("dbt_required_selector_terms")?)?, + dbt_skip_reason: reason_value(&plan.getattr("dbt_skip_reason")?)?, + sqlbuild_argvs: argvs(&plan.getattr("sqlbuild_command_argvs")?)?, + sqlbuild_models: strings(&selection.getattr("sqlbuild_model_names")?)?, + sqlbuild_skip_reason: reason_value(&plan.getattr("sqlbuild_skip_reason")?)?, + anchors, + path_translations, + warnings: strings(&plan.getattr("warnings")?)?, + }) +} + +fn tuple_of<'py>(value: Bound<'py, PyAny>) -> PyResult> { + Ok(PyTuple::new(value.py(), value.try_iter()?.collect::>>()?)?.into_any()) +} + +fn tuple_of_tuples<'py>(value: Bound<'py, PyAny>) -> PyResult> { + let items: Vec> = value + .try_iter()? + .map(|item| tuple_of(item?)) + .collect::>()?; + Ok(PyTuple::new(value.py(), items)?.into_any()) +} + +fn empty_or<'py>(value: Bound<'py, PyAny>) -> Bound<'py, PyAny> { + if value.is_none() { + return PyTuple::empty(value.py()).into_any(); + } + value +} + +fn skip_reason<'py>(py: Python<'py>, has_work: bool, member: &str) -> PyResult> { + if has_work { + return Ok(py.None().into_bound(py)); + } + imported(py, DBT_TYPES, "DbtInteropSkipReason")?.getattr(member) +} + +/// Build a display-ready selection plan without dbt model state. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_dbt_interop_plan<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "build_dbt_interop_plan", options, BUILD_OPTIONS)?; + let command: Bound<'py, PyAny> = + imported(py, DBT_TYPES, "DbtInteropCommand")?.call1((arguments.value("command")?,))?; + let nodes: Bound<'py, PyAny> = tuple_of(arguments.value("dbt_ls_nodes")?)?; + let mut selected: Vec = Vec::new(); + for node in nodes.try_iter()? { + selected.push(node?.getattr("unique_id")?.extract()?); + } + selected.sort(); + let selection: Bound<'py, PyAny> = arguments.value("selection")?; + let dbt_has_work: bool = + !selected.is_empty() || selection.getattr("dbt_required_unique_ids")?.is_truthy()?; + let sqlbuild_has_work: bool = selection.getattr("sqlbuild_model_names")?.is_truthy()?; + let warnings: Vec = dbt_interop_plan_warnings( + strings(&empty_or(arguments.value("warnings")?))?, + dbt_has_work || sqlbuild_has_work, + ); + call_with( + &imported(py, DBT_MODELS, "DbtInteropPlan")?, + &[ + ("command", command), + ( + "dbt_command_argv", + tuple_of(arguments.value("dbt_command_argv")?)?, + ), + ("dbt_selected_nodes", nodes), + ( + "dbt_selected_unique_ids", + PyTuple::new(py, selected)?.into_any(), + ), + ( + "sqlbuild_command_argvs", + tuple_of_tuples(arguments.value("sqlbuild_command_argvs")?)?, + ), + ("selection", selection), + ( + "sqlbuild_plan_output", + arguments.value("sqlbuild_plan_output")?, + ), + ( + "dbt_required_selector_terms", + tuple_of(empty_or(arguments.value("dbt_required_selector_terms")?))?, + ), + ( + "supplemental_dbt_command_argvs", + tuple_of_tuples(empty_or(arguments.value("supplemental_dbt_command_argvs")?))?, + ), + ( + "dbt_skip_reason", + skip_reason(py, dbt_has_work, "NO_DBT_WORK")?, + ), + ( + "sqlbuild_skip_reason", + skip_reason(py, sqlbuild_has_work, "NO_SQLBUILD_WORK")?, + ), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ], + ) +} + +fn display_limit(options: &Bound<'_, PyAny>) -> PyResult { + Ok(DisplayLimit { + max_entries: options.getattr("max_entries_per_section")?.extract()?, + overflow_flag: options.getattr("overflow_flag")?.extract()?, + }) +} + +/// Render a dbt interop plan in the requested CLI output format. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_dbt_interop_plan(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "render_dbt_interop_plan", options, RENDER_OPTIONS)?; + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let text: DbtInteropPlanText = plan_text(&plan)?; + if arguments.flag("json_output")? { + return Ok(format_dbt_interop_plan_json(&text)); + } + let mut display: Bound<'_, PyAny> = arguments.value("display_options")?; + if !display.is_truthy()? { + display = imported(py, PRESENTATION_MODELS, "DisplayOptions")?.call0()?; + } + let use_color: bool = arguments.flag("use_color")?; + let plan_output: Bound<'_, PyAny> = plan.getattr("sqlbuild_plan_output")?; + let sqlbuild_plan: Option = + if text.sqlbuild_skip_reason.is_some() || plan_output.is_none() { + None + } else { + Some( + call_with( + &imported(py, NATIVE_MODULE, "format_plan")?, + &[ + ("plan", plan_output), + ( + "use_color", + PyBool::new(py, use_color).to_owned().into_any(), + ), + ("display_options", display.clone()), + ], + )? + .extract()?, + ) + }; + Ok(format!( + "\n{}", + format_dbt_interop_plan( + &text, + sqlbuild_plan.as_deref(), + (&display_limit(&display)?, use_color) + ) + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_dbt_interop_plan, module)?)?; + module.add_function(wrap_pyfunction!(render_dbt_interop_plan, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs new file mode 100644 index 0000000000..678ca86dcd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs @@ -0,0 +1,160 @@ +//! dbt profile normalization and the project config `sqb dbt init` writes. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::dbt_profile::errors::DbtProfileError; +use sqlbuild_config::dbt_profile::main::dbt_init_project_toml::dbt_init_project_toml; +use sqlbuild_config::dbt_profile::main::normalize_dbt_profile::normalize_dbt_profile; +use sqlbuild_config::dbt_profile::models::{ + ConnectionKey, ConnectionSource, DbtInitConfig, NormalizedProfile, ProfileEntry, ProfileValue, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const DBT_ERRORS_MODULE: &str = "sqlbuild.integrations.dbt.exceptions"; +const INIT_TOML_OPTIONS: (&[&str], &[&str]) = ( + &[ + "project_name", + "adapter", + "target_name", + "profile_name", + "dbt_project_dir", + "profiles_dir", + "target_path", + "target_schema", + "target_database", + ], + &[], +); + +fn raised(py: Python<'_>, error: DbtProfileError) -> PyErr { + match imported(py, DBT_ERRORS_MODULE, "DbtProfileError") + .and_then(|class| class.call1((error.0,))) + { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +fn profile_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(ProfileValue::Null); + } + if value.is_instance_of::() { + return Ok(ProfileValue::Bool(value.is_truthy()?)); + } + if value.is_instance_of::() { + return Ok(ProfileValue::Text(value.extract()?)); + } + Ok(ProfileValue::Other) +} + +fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { + match value { + Some(text) => PyString::new(py, text).into_any(), + None => py.None().into_bound(py), + } +} + +/// `NormalizedDbtProfileConnection` with the connection built from `output`'s values. +fn normalized_connection<'py>( + output: &Bound<'py, PyDict>, + profile: &NormalizedProfile, +) -> PyResult> { + let py: Python<'py> = output.py(); + let items: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = output.iter().collect(); + let connection: Bound<'py, PyDict> = PyDict::new(py); + for (key, source) in &profile.connection { + let key: Bound<'py, PyAny> = match key { + ConnectionKey::Name(name) => PyString::new(py, name).into_any(), + ConnectionKey::Entry(index) => items[*index].0.clone(), + }; + let value: Bound<'py, PyAny> = match source { + ConnectionSource::Entry(index) => items[*index].1.clone(), + ConnectionSource::Text(text) => PyString::new(py, text).into_any(), + }; + connection.set_item(key, value)?; + } + call_with( + &imported(py, DBT_MODELS, "NormalizedDbtProfileConnection")?, + &[ + ("adapter", PyString::new(py, profile.adapter).into_any()), + ("connection", connection.into_any()), + ( + "target_schema", + optional_text(py, profile.target_schema.as_deref()), + ), + ( + "target_database", + optional_text(py, profile.target_database.as_deref()), + ), + ("warnings", PyTuple::new(py, &profile.warnings)?.into_any()), + ], + ) +} + +/// Normalize rendered dbt profile output for SQLBuild. +#[pyfunction] +#[pyo3(signature = (*, resolved))] +fn normalize_dbt_profile_connection<'py>( + py: Python<'py>, + resolved: Bound<'py, PyAny>, +) -> PyResult> { + let output: Bound<'py, PyDict> = PyDict::new(py); + output.update(&resolved.getattr("output")?.cast_into()?)?; + let mut entries: Vec = Vec::new(); + for (key, value) in output.iter() { + entries.push(ProfileEntry { + key: if key.is_instance_of::() { + Some(key.extract()?) + } else { + None + }, + value: profile_value(&value)?, + }); + } + let profile: NormalizedProfile = + normalize_dbt_profile(&entries).map_err(|error| raised(py, error))?; + normalized_connection(&output, &profile) +} + +fn text_argument(arguments: &KeywordArguments<'_>, name: &str) -> PyResult { + arguments.value(name)?.extract() +} + +/// The `sqlbuild_project.toml` text `sqb dbt init` writes. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_dbt_init_project_toml( + py: Python<'_>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "build_dbt_init_project_toml", + options, + INIT_TOML_OPTIONS, + )?; + Ok(dbt_init_project_toml(&DbtInitConfig { + project_name: text_argument(&arguments, "project_name")?, + adapter: text_argument(&arguments, "adapter")?, + target_name: text_argument(&arguments, "target_name")?, + profile_name: text_argument(&arguments, "profile_name")?, + dbt_project_dir: text_argument(&arguments, "dbt_project_dir")?, + profiles_dir: text_argument(&arguments, "profiles_dir")?, + target_path: text_argument(&arguments, "target_path")?, + target_schema: arguments.value("target_schema")?.extract()?, + target_database: arguments.value("target_database")?.extract()?, + })) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(normalize_dbt_profile_connection, module)?)?; + module.add_function(wrap_pyfunction!(build_dbt_init_project_toml, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs new file mode 100644 index 0000000000..2cd3a2ed8b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -0,0 +1,11 @@ +//! Register the static `sqb dbt` interop helpers. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::dbt_interop::{manifest, plan, profile}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + manifest::register(module)?; + plan::register(module)?; + profile::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs new file mode 100644 index 0000000000..5dfdec4811 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs @@ -0,0 +1 @@ +pub(crate) mod redaction; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs new file mode 100644 index 0000000000..603adb3049 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs @@ -0,0 +1,84 @@ +//! Secret-safe diagnostic text and structured logging field values. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyStringMethods, PyTuple, + PyTypeMethods, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::redaction::main::redact_text::redact_text; +use sqlbuild_core::redaction::main::secret_field_name::secret_field_name; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const REDACTED: &str = "[REDACTED]"; + +/// Obvious named secrets and URI passwords redacted from diagnostic text. +#[pyfunction] +#[pyo3(signature = (value))] +fn redact_diagnostic_text(value: &str) -> String { + redact_text(value) +} + +fn redacted_mapping<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let redacted: Bound<'py, PyDict> = PyDict::new(py); + for item in value.call_method0("items")?.try_iter()? { + let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let name: String = key.str()?.extract()?; + redacted.set_item(&name, redacted_value(&name, &item)?)?; + } + Ok(redacted.into_any()) +} + +/// A persistence-safe value for one structured logging field. +fn redacted_value<'py>(name: &str, value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if secret_field_name(name) { + return Ok(PyString::new(py, REDACTED).into_any()); + } + if let Ok(text) = value.downcast::() { + return Ok(PyString::new(py, &redact_text(text.to_str()?)).into_any()); + } + let scalar: bool = value.is_none() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::(); + if scalar { + return Ok(value.clone()); + } + if value.is_instance(&imported(py, "collections.abc", "Mapping")?)? { + return redacted_mapping(value); + } + if value.is_instance_of::() || value.is_instance_of::() { + let items: Vec> = value + .try_iter()? + .map(|item| redacted_value(name, &item?)) + .collect::>()?; + return Ok(PyList::new(py, items)?.into_any()); + } + let rendered: PyResult = value.str().and_then(|text| text.extract()); + Ok(match rendered { + Ok(text) => PyString::new(py, &redact_text(&text)).into_any(), + Err(_) => { + let type_name: String = value.get_type().name()?.extract()?; + PyString::new(py, &format!("<{type_name}>")).into_any() + } + }) +} + +/// A persistence-safe value for one structured logging field. +#[pyfunction] +#[pyo3(signature = (*, name, value))] +fn redact_diagnostic_value<'py>( + name: &str, + value: &Bound<'py, PyAny>, +) -> PyResult> { + redacted_value(name, value) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(redact_diagnostic_text, module)?)?; + module.add_function(wrap_pyfunction!(redact_diagnostic_value, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs new file mode 100644 index 0000000000..164f2b106b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -0,0 +1,524 @@ +//! `sqb diff`: validate, discover, run a model or raw-query comparison and publish its output. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyTuple}; +use sqlbuild_cli::diff_report::errors::DiffUserError; +use sqlbuild_cli::diff_report::main::diff_exit_code::diff_exit_code; +use sqlbuild_cli::diff_report::main::effective_max_examples::effective_max_examples; +use sqlbuild_cli::diff_report::main::is_incomplete_query_failure::is_incomplete_query_failure; +use sqlbuild_cli::diff_report::main::is_query_diff_request::is_query_diff_request; +use sqlbuild_cli::diff_report::main::size_guard_report::size_guard_report; +use sqlbuild_cli::diff_report::main::validate_diff_request::validate_diff_request; +use sqlbuild_cli::diff_report::models::{ + DiffModelFacts, DiffRequestFacts, FullDiffModelFacts, FullDiffSideFacts, SizeGuardReport, +}; +use sqlbuild_cli::diff_report::types::QueryDiffPhase; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::diff_report::facts::diff_facts; +use crate::bindings::_helpers::diff_report::query::{ + execute_query_diff, execution_options, prepare_query_diff, project_adapter, +}; +use crate::bindings::_helpers::diff_report::registration::publish_diff_output; +use crate::bindings::_helpers::diff_report::support::{ + EXCEPTIONS_MODULE, call, caused, cli_error, error_text, is_exception, matches_class, none, + object, request_facts, user_error_parts, +}; +use crate::bindings::_helpers::frontier::encoder::emit_error; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const CONFIGURATION_MODULE: &str = "sqlbuild.compiler.discovery.main._discover_configuration"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const CONTRACT_CONSTANTS_MODULE: &str = "sqlbuild.spec.contracts.constants"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXECUTE_MODULE: &str = "sqlbuild.executor.diff.main.execute"; +const DIFF_MODELS_MODULE: &str = "sqlbuild.executor.diff.models"; +const ADAPTER_ERRORS: (&str, &str) = ("sqlbuild.adapter.contract.exceptions", "AdapterUserError"); +const EXECUTOR_INPUT_ERRORS: (&str, &str) = + ("sqlbuild.errors.contracts.exceptions", "ExecutorInputError"); +const SIZE_GUARD_ERRORS: (&str, &str) = ( + "sqlbuild.executor.diff.exceptions", + "FullDiffSizeGuardError", +); +const OUTCOME_ERRORS: (&str, &str) = (EXCEPTIONS_MODULE, "QueryDiffOutcomeError"); +const USER_ERRORS: (&str, &str) = (EXCEPTIONS_MODULE, "CliUserError"); +const ROW_DIFF_KEY_ERROR_PREFIX: &str = "row diff "; +const ROW_DIFF_KEY_ERROR_FRAGMENT: &str = " unique_key values"; + +fn user(message: &str, code: &str, help: Option<&str>) -> DiffUserError { + DiffUserError { + message: message.to_owned(), + code: code.to_owned(), + help: help.map(str::to_owned), + } +} + +/// `CliUserError` from a pure rejection. +fn rejected(py: Python<'_>, error: &DiffUserError) -> PyErr { + cli_error(py, "CliUserError", error) +} + +/// The project directory and discovered inputs, after the request's flags validate. +fn resolve_invocation<'py>( + request: &Bound<'py, PyAny>, + facts: &DiffRequestFacts, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = request.py(); + let query_mode: bool = is_query_diff_request(facts); + if let Err(error) = validate_diff_request(facts) { + return Err(cli_error( + py, + if query_mode { + "QueryDiffIncompleteError" + } else { + "CliUserError" + }, + &error, + )); + } + let project_dir: Bound<'py, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => imported(py, "pathlib", "Path")?.call_method0("cwd")?, + }; + let discovered: Bound<'py, PyAny> = if query_mode { + call( + &imported(py, CONFIGURATION_MODULE, "discover_project_configuration")?, + &[("project_dir", &project_dir)], + )? + } else { + call( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", &project_dir)], + )? + }; + call( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", &request.getattr("select")?), + ("exclude", &request.getattr("exclude")?), + ("discovered_inputs", &discovered), + ], + )?; + if facts.select.is_empty() && !query_mode { + return Err(rejected( + py, + &user("diff requires --select in v1", "C204", None), + )); + } + Ok((project_dir, discovered)) +} + +/// `message`, `code` and `help` re-raised as an incomplete or failed query-diff outcome. +fn query_outcome_error( + py: Python<'_>, + incomplete: bool, + parts: &DiffUserError, + cause: PyErr, +) -> PyErr { + let class: &str = if incomplete { + "QueryDiffIncompleteError" + } else { + "QueryDiffExecutionError" + }; + caused(py, cli_error(py, class, parts), cause) +} + +fn execution_failure(py: Python<'_>, code: &str, cause: PyErr) -> PyResult { + let parts: DiffUserError = user(&error_text(py, &cause)?, code, None); + Ok(query_outcome_error(py, false, &parts, cause)) +} + +/// Map a raw-query execution failure to its incomplete or execution-failure outcome. +fn query_execution_error(py: Python<'_>, error: PyErr) -> PyResult { + if matches_class(py, &error, OUTCOME_ERRORS)? || !is_exception(py, &error) { + return Ok(error); + } + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + if matches_class(py, &error, EXECUTOR_INPUT_ERRORS)? { + let code: String = match value.getattr_opt("code")? { + Some(code) => code.str()?.extract()?, + None => "C254".to_owned(), + }; + let help: Option = match value.getattr_opt("help")? { + Some(help) if !help.is_none() => Some(help.extract()?), + _ => None, + }; + let parts: DiffUserError = DiffUserError { + message: error_text(py, &error)?, + code, + help, + }; + return Ok(query_outcome_error(py, true, &parts, error)); + } + if matches_class(py, &error, USER_ERRORS)? { + let parts: DiffUserError = user_error_parts(&value)?; + let incomplete: bool = is_incomplete_query_failure(QueryDiffPhase::Execution, &parts.code); + return Ok(query_outcome_error(py, incomplete, &parts, error)); + } + if matches_class(py, &error, ADAPTER_ERRORS)? { + let parts: DiffUserError = user_error_parts(&value)?; + let incomplete: bool = parts.message.starts_with(ROW_DIFF_KEY_ERROR_PREFIX) + && parts.message.contains(ROW_DIFF_KEY_ERROR_FRAGMENT); + return Ok(query_outcome_error(py, incomplete, &parts, error)); + } + execution_failure(py, "C255", error) +} + +fn run_query_diff( + request: &Bound<'_, PyAny>, + invocation: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = request.py(); + let preparation: Bound<'_, PyAny> = match prepare_query_diff(request, invocation) { + Ok(preparation) => preparation, + Err(error) if matches_class(py, &error, USER_ERRORS)? => { + let parts: DiffUserError = user_error_parts(&error.value(py).clone().into_any())?; + let incomplete: bool = + is_incomplete_query_failure(QueryDiffPhase::Preparation, &parts.code); + return Err(query_outcome_error(py, incomplete, &parts, error)); + } + Err(error) if is_exception(py, &error) => { + return Err(execution_failure(py, "C253", error)?); + } + Err(error) => return Err(error), + }; + let (result, phases) = match execute_query_diff(request, &preparation) { + Ok(outcome) => outcome, + Err(error) => return Err(query_execution_error(py, error)?), + }; + let models: Vec = diff_facts(&result)?; + let schema_only: bool = request.getattr("schema_only")?.is_truthy()?; + let (outcome, exit_code) = diff_exit_code(&models, !schema_only); + let published: PyResult<()> = publish_diff_output( + request, + ( + &preparation.getattr("left_label")?.extract::()?, + &preparation.getattr("right_label")?.extract::()?, + ), + ( + preparation + .getattr("effective_max_column_examples")? + .extract()?, + preparation + .getattr("effective_max_row_only_examples")? + .extract()?, + ), + (&models, Some(phases), Some(outcome)), + ); + match published { + Ok(()) => Ok(exit_code), + Err(error) if is_exception(py, &error) => { + let parts: DiffUserError = user( + &format!( + "failed to publish query diff output: {}", + error_text(py, &error)? + ), + "C257", + None, + ); + Err(query_outcome_error(py, false, &parts, error)) + } + Err(error) => Err(error), + } +} + +fn target_max_full_rows<'py>( + discovered: &Bound<'py, PyAny>, + target: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let configured: Bound<'py, PyAny> = call( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ("project_config", &discovered.getattr("project_config")?), + ("local_config", &discovered.getattr("local_config")?), + ("target_name", target), + ], + )? + .getattr("diff")? + .getattr("max_full_rows")?; + if configured.is_none() { + return imported(py, CONTRACT_CONSTANTS_MODULE, "DEFAULT_DIFF_MAX_FULL_ROWS"); + } + if configured.is_instance_of::() && !configured.is_instance_of::() { + return Ok(configured); + } + Ok(none(py)) +} + +/// The active target when only FROM was given, rejecting a missing or identical target. +fn diff_to_target<'py>( + request: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, + from_target: &str, +) -> PyResult> { + let py: Python<'py> = request.py(); + let to_name: Bound<'py, PyAny> = request.getattr("to_name")?; + if !to_name.is_none() { + return Ok(to_name); + } + let active: Bound<'py, PyAny> = call( + &imported(py, TARGET_NAME_MODULE, "resolve_target_name")?, + &[ + ("project_config", &discovered.getattr("project_config")?), + ("local_config", &discovered.getattr("local_config")?), + ("selected_target", &none(py)), + ], + )?; + if active.is_none() { + return Err(rejected( + py, + &user( + "model diff requires FROM:TO when the project has no active target", + "C271", + Some("set default_target, or pass both targets as FROM:TO"), + ), + )); + } + if active.extract::()? == from_target { + return Err(rejected( + py, + &user( + &format!( + "diff FROM '{from_target}' is the active target, so there is nothing to compare" + ), + "C272", + Some("pass both targets as FROM:TO"), + ), + )); + } + Ok(active) +} + +fn side_facts(side: &Bound<'_, PyAny>) -> PyResult { + let detail: Bound<'_, PyAny> = side.getattr("detail")?; + Ok(FullDiffSideFacts { + target: side.getattr("target")?.extract()?, + relation: side.getattr("relation")?.extract()?, + row_count: side.getattr("row_count")?.extract()?, + max_rows: side.getattr("max_rows")?.extract()?, + exceeds_limit: side.getattr("exceeds_limit")?.is_truthy()?, + detail: if detail.is_none() { + None + } else { + Some(detail.str()?.extract()?) + }, + }) +} + +/// `DiffSizeGuardError` (C270) for the guard's blocked models. +fn size_guard_error( + py: Python<'_>, + facts: &DiffRequestFacts, + targets: (&str, &str), + error: PyErr, +) -> PyResult { + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + let mut blocked: Vec = Vec::new(); + for model in value.getattr("blocked")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + blocked.push(FullDiffModelFacts { + name: model.getattr("name")?.extract()?, + has_cursor: model.getattr("has_cursor")?.is_truthy()?, + left: side_facts(&model.getattr("left")?)?, + right: side_facts(&model.getattr("right")?)?, + }); + } + let message: String = value.getattr("message")?.extract()?; + let report: SizeGuardReport = size_guard_report(facts, targets, &message, &blocked); + let details_text: String = dumps( + &report.details, + &JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + ) + .map_err(emit_error)?; + let details: Bound<'_, PyAny> = imported(py, "json", "loads")?.call1((details_text,))?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("details", details)?; + keywords.set_item("code", "C270")?; + keywords.set_item("help", report.help)?; + let replacement: Bound<'_, PyAny> = imported(py, EXCEPTIONS_MODULE, "DiffSizeGuardError")? + .call((report.message.as_str(),), Some(&keywords))?; + Ok(caused(py, PyErr::from_value(replacement), error)) +} + +fn run_model_diff( + request: &Bound<'_, PyAny>, + facts: &DiffRequestFacts, + (project_dir, discovered): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = request.py(); + let Some(from_name) = facts.from_name.as_deref() else { + return Err(rejected( + py, + &user("model diff requires FROM or FROM:TO", "C224", None), + )); + }; + let to_target: Bound<'_, PyAny> = diff_to_target(request, discovered, from_name)?; + let to_name: String = to_target.extract()?; + let targets: Bound<'_, PyAny> = discovered.getattr("project_config")?.getattr("targets")?; + if !targets.contains(from_name)? { + return Err(rejected( + py, + &user( + &format!("unknown diff FROM target '{from_name}'"), + "C205", + None, + ), + )); + } + if !targets.contains(&to_name)? { + return Err(rejected( + py, + &user(&format!("unknown diff TO target '{to_name}'"), "C206", None), + )); + } + let from_target: Bound<'_, PyAny> = object(py, from_name)?; + let (_, adapter) = project_adapter(discovered, project_dir)?; + let cli_vars: Bound<'_, PyAny> = request.getattr("cli_vars")?; + let connection_config: Bound<'_, PyAny> = call( + &imported(py, NATIVE_MODULE, "resolve_target_connection_config")?, + &[ + ("discovered_inputs", discovered), + ("project_dir", project_dir), + ("target_name", &to_target), + ("cli_vars", &cli_vars), + ], + )?; + let resolver: Bound<'_, PyAny> = call( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", project_dir), + ("discovered_inputs", discovered), + ], + )?; + let pipeline: Bound<'_, PyTuple> = call( + &imported(py, NATIVE_MODULE, "run_diff_pipeline")?, + &[ + ("discovered_inputs", discovered), + ("adapter", &adapter), + ("from_target", &from_target), + ("to_target", &to_target), + ("resolved_connection", &connection_config), + ("no_sql_validation", &request.getattr("no_sql_validation")?), + ("select", &request.getattr("select")?), + ("exclude", &request.getattr("exclude")?), + ("cli_vars", &cli_vars), + ("external_sql_reference_resolver", &resolver), + ], + )? + .downcast_into()?; + let selected: Bound<'_, PyAny> = pipeline.get_item(2)?; + if !selected.is_truthy()? { + return Err(rejected( + py, + &user( + "no diffable models found in the selected scope", + "C207", + None, + ), + )); + } + let verbose: bool = request.getattr("verbose")?.is_truthy()?; + let column_examples: i64 = effective_max_examples(facts.max_column_examples, verbose); + let row_only_examples: i64 = effective_max_examples(facts.max_row_only_examples, verbose); + let limits: Bound<'_, PyAny> = if facts.full || facts.schema_only || facts.bounded.is_some() { + none(py) + } else { + call( + &imported(py, DIFF_MODELS_MODULE, "FullDiffSizeLimits")?, + &[ + ("left_target", &from_target), + ("right_target", &to_target), + ( + "left_max_rows", + &target_max_full_rows(discovered, &from_target)?, + ), + ( + "right_max_rows", + &target_max_full_rows(discovered, &to_target)?, + ), + ], + )? + }; + let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (&connection_config,))?; + let extras: [(&str, Bound<'_, PyAny>); 5] = [ + ("bounded", request.getattr("bounded")?), + ("max_column_examples", object(py, column_examples)?), + ("max_row_only_examples", object(py, row_only_examples)?), + ("max_models", request.getattr("max_models")?), + ("full_size_limits", limits), + ]; + let extra: Vec<(&str, &Bound<'_, PyAny>)> = + extras.iter().map(|(key, value)| (*key, value)).collect(); + let executed: PyResult> = + execution_options(request, &extra).and_then(|options| { + call( + &imported(py, EXECUTE_MODULE, "execute_diff")?, + &[ + ("adapter", &adapter), + ("connection", &connection), + ("left_project", &pipeline.get_item(0)?), + ("right_project", &pipeline.get_item(1)?), + ("selected_names", &selected), + ("options", &options), + ], + ) + }); + let executed: PyResult> = match executed { + Err(error) if matches_class(py, &error, SIZE_GUARD_ERRORS)? => { + Err(size_guard_error(py, facts, (from_name, &to_name), error)?) + } + other => other, + }; + adapter.call_method1("close", (&connection,))?; + let result: Bound<'_, PyAny> = executed?; + let models: Vec = diff_facts(&result)?; + publish_diff_output( + request, + (from_name, &to_name), + (column_examples, row_only_examples), + (&models, None, None), + )?; + Ok(diff_exit_code(&models, false).1) +} + +/// Run one `sqb diff` request; the exit code. +#[pyfunction] +#[pyo3(signature = (*, request))] +pub(crate) fn run_diff_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let facts: DiffRequestFacts = request_facts(request)?; + let query_mode: bool = is_query_diff_request(&facts); + let (project_dir, discovered) = match resolve_invocation(request, &facts) { + Ok(invocation) => invocation, + Err(error) + if query_mode + && !matches_class(py, &error, OUTCOME_ERRORS)? + && is_exception(py, &error) => + { + return Err(execution_failure(py, "C252", error)?); + } + Err(error) => return Err(error), + }; + if query_mode { + return run_query_diff(request, (&project_dir, &discovered)); + } + run_model_diff(request, &facts, (&project_dir, &discovered)) +} + +/// Materialize, compare and clean one prepared raw-query pair; the diff result. +#[pyfunction] +#[pyo3(name = "execute_query_diff", signature = (*, request, preparation))] +pub(crate) fn execute_prepared_query_diff<'py>( + request: &Bound<'py, PyAny>, + preparation: &Bound<'py, PyAny>, +) -> PyResult> { + execute_query_diff(request, preparation).map(|(result, _)| result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs new file mode 100644 index 0000000000..56bd8aeea2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs @@ -0,0 +1,273 @@ +//! Python diff results read as plain diff facts. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, + PyStringMethods, PyTuple, +}; +use sqlbuild_cli::diff_report::models::{ + DiffChangedCell, DiffColumnFacts, DiffCoverageFacts, DiffModelFacts, DiffRowFacts, + DiffSampleRow, DiffValue, +}; +use sqlbuild_cli::diff_report::types::DiffNamedValue; +use sqlbuild_core::json::main::python_float_repr::python_float_repr; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +fn text_of(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.str()?.to_string_lossy().into_owned()) +} + +fn strings(value: &Bound<'_, PyAny>) -> PyResult> { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(text_of(&item?)?); + } + Ok(items) +} + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + text_of(&value).map(Some) +} + +/// A JSON object key as `json.dumps` writes it. +fn json_key(key: &Bound<'_, PyAny>) -> PyResult { + if let Ok(flag) = key.downcast::() { + return Ok(if flag.is_true() { "true" } else { "false" }.to_owned()); + } + if key.is_none() { + return Ok("null".to_owned()); + } + if key.is_instance_of::() { + return Ok(python_float_repr(key.extract()?)); + } + text_of(key) +} + +/// One value as `json.dumps(default=...)` encodes it: dates as ISO text, other objects as `str()`. +pub(crate) fn json_of(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(JsonValue::Null); + } + if let Ok(text) = value.downcast::() { + return Ok(JsonValue::String(text.to_string_lossy().into_owned())); + } + if let Ok(flag) = value.downcast::() { + return Ok(JsonValue::Bool(flag.is_true())); + } + if value.is_instance_of::() { + let decimal: String = value + .call_method0("__index__")? + .str()? + .to_string_lossy() + .into_owned(); + if let Some(integer) = JsonInteger::parse(&decimal) { + return Ok(JsonValue::Integer(integer)); + } + } + if value.is_instance_of::() { + return Ok(JsonValue::Float(value.extract()?)); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(json_of(&item?)?); + } + return Ok(JsonValue::Array(items)); + } + if let Ok(mapping) = value.downcast::() { + let mut entries: Vec<(String, JsonValue)> = Vec::new(); + for (key, item) in mapping.iter() { + entries.push((json_key(&key)?, json_of(&item)?)); + } + return Ok(JsonValue::Object(entries)); + } + let py: Python<'_> = value.py(); + if value.is_instance(&imported(py, "datetime", "date")?)? { + return Ok(JsonValue::String(text_of( + &value.call_method0("isoformat")?, + )?)); + } + Ok(JsonValue::String(text_of(value)?)) +} + +fn diff_value(value: &Bound<'_, PyAny>) -> PyResult { + Ok(DiffValue { + text: text_of(value)?, + json: json_of(value)?, + }) +} + +fn optional_value(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + diff_value(&value).map(Some) +} + +fn named_values(value: &Bound<'_, PyAny>) -> PyResult> { + let mut pairs: Vec = Vec::new(); + for pair in value.try_iter()? { + let (name, item): (Bound<'_, PyAny>, Bound<'_, PyAny>) = pair?.extract()?; + pairs.push((text_of(&name)?, diff_value(&item)?)); + } + Ok(pairs) +} + +fn key_samples(value: &Bound<'_, PyAny>) -> PyResult>> { + let mut samples: Vec> = Vec::new(); + for sample in value.try_iter()? { + samples.push(named_values(&sample?)?); + } + Ok(samples) +} + +fn coverage(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(DiffCoverageFacts { + row_count: value.getattr("row_count")?.extract()?, + minimum_cursor: optional_value(value.getattr("minimum_cursor")?)?, + maximum_cursor: optional_value(value.getattr("maximum_cursor")?)?, + })) +} + +fn column(value: &Bound<'_, PyAny>) -> PyResult { + let tolerance: Bound<'_, PyAny> = value.getattr("tolerance")?; + Ok(DiffColumnFacts { + name: text_of(&value.getattr("name")?)?, + mismatched_count: value.getattr("mismatched_count")?.extract()?, + tolerance: if tolerance.is_none() { + None + } else { + Some(( + optional_text(tolerance.getattr("absolute")?)?, + optional_text(tolerance.getattr("relative")?)?, + )) + }, + }) +} + +fn coverage_differs(rows: &Bound<'_, PyAny>) -> PyResult { + let left: Bound<'_, PyAny> = rows.getattr("left_coverage")?; + let right: Bound<'_, PyAny> = rows.getattr("right_coverage")?; + if left.is_none() || right.is_none() { + return Ok(false); + } + Ok(left + .getattr("minimum_cursor")? + .ne(right.getattr("minimum_cursor")?)? + || left + .getattr("maximum_cursor")? + .ne(right.getattr("maximum_cursor")?)?) +} + +fn row_facts(rows: Bound<'_, PyAny>) -> PyResult> { + if rows.is_none() { + return Ok(None); + } + let count = |name: &str| -> PyResult { rows.getattr(name)?.extract() }; + let mut columns: Vec = Vec::new(); + for item in rows.getattr("column_results")?.try_iter()? { + columns.push(column(&item?)?); + } + let sampling: Bound<'_, PyAny> = rows.getattr("sampling")?; + Ok(Some(DiffRowFacts { + left_count: count("left_count")?, + right_count: count("right_count")?, + joined_count: count("joined_count")?, + equal_count: count("equal_count")?, + unequal_count: count("unequal_count")?, + left_only_count: count("left_only_count")?, + right_only_count: count("right_only_count")?, + columns, + population_count: count("population_count")?, + compared_count: count("compared_count")?, + sampling: if sampling.is_none() { + None + } else { + Some(( + sampling.getattr("row_limit")?.extract()?, + sampling.getattr("seed")?.extract()?, + )) + }, + left_coverage: coverage(rows.getattr("left_coverage")?)?, + right_coverage: coverage(rows.getattr("right_coverage")?)?, + cursor_column: optional_text(rows.getattr("cursor_column")?)?, + cursor_start: optional_value(rows.getattr("cursor_start")?)?, + cursor_end: optional_value(rows.getattr("cursor_end")?)?, + exhaustive: rows.getattr("is_exhaustive")?.is_truthy()?, + coverage_differs: coverage_differs(&rows)?, + })) +} + +fn sample_row(value: &Bound<'_, PyAny>) -> PyResult { + let mut changed_cells: Vec = Vec::new(); + for cell in value.getattr("changed_cells")?.try_iter()? { + let cell: Bound<'_, PyAny> = cell?; + changed_cells.push(DiffChangedCell { + name: text_of(&cell.getattr("name")?)?, + left: diff_value(&cell.getattr("left_value")?)?, + right: diff_value(&cell.getattr("right_value")?)?, + }); + } + Ok(DiffSampleRow { + key_values: named_values(&value.getattr("key_values")?)?, + changed_cells, + }) +} + +fn model_facts(model: &Bound<'_, PyAny>) -> PyResult { + let schema: Bound<'_, PyAny> = model.getattr("schema_result")?; + let names = |attribute: &str| -> PyResult> { + let mut names: Vec = Vec::new(); + for column in schema.getattr(attribute)?.try_iter()? { + names.push(text_of(&column?.getattr("name")?)?); + } + Ok(names) + }; + let mut type_changes: Vec<(String, String, String)> = Vec::new(); + for pair in schema.getattr("type_changed_columns")?.try_iter()? { + let (left, right): (Bound<'_, PyAny>, Bound<'_, PyAny>) = pair?.extract()?; + type_changes.push(( + text_of(&left.getattr("name")?)?, + text_of(&left.getattr("type")?)?, + text_of(&right.getattr("type")?)?, + )); + } + let mut unequal_samples: Vec = Vec::new(); + for sample in model.getattr("unequal_row_samples")?.try_iter()? { + unequal_samples.push(sample_row(&sample?)?); + } + Ok(DiffModelFacts { + name: text_of(&model.getattr("name")?)?, + left_relation: text_of(&model.getattr("left_relation")?)?, + right_relation: text_of(&model.getattr("right_relation")?)?, + added_columns: names("added_columns")?, + removed_columns: names("removed_columns")?, + type_changes, + unique_key: strings(&model.getattr("unique_key")?)?, + rows: row_facts(model.getattr("row_result")?)?, + unequal_samples, + left_only_keys: key_samples(&model.getattr("left_only_key_samples")?)?, + right_only_keys: key_samples(&model.getattr("right_only_key_samples")?)?, + bounded_fallback: model.getattr("bounded_fallback")?.is_truthy()?, + excluded_columns: strings(&model.getattr("excluded_columns")?)?, + input_kind: text_of(&model.getattr("input_kind")?)?, + unkeyed: model.getattr("unkeyed")?.is_truthy()?, + }) +} + +/// Every model result of a `DiffExecutionResult`. +pub(crate) fn diff_facts(result: &Bound<'_, PyAny>) -> PyResult> { + let mut models: Vec = Vec::new(); + for model in result.getattr("model_results")?.try_iter()? { + models.push(model_facts(&model?)?); + } + Ok(models) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs new file mode 100644 index 0000000000..a2dea05e2a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod command; +pub(crate) mod facts; +pub(crate) mod query; +pub(crate) mod registration; +pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs new file mode 100644 index 0000000000..6efc6275fc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -0,0 +1,772 @@ +//! Raw-query `sqb diff`: preparation, artifact materialization, comparison and cleanup. + +use std::time::Instant; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_cli::diff_report::errors::DiffUserError; +use sqlbuild_cli::diff_report::main::effective_max_examples::effective_max_examples; +use sqlbuild_cli::diff_report::main::query_column_problem::query_column_problem; +use sqlbuild_cli::diff_report::main::read_only_query_problem::read_only_query_problem; +use sqlbuild_cli::diff_report::models::QueryShapeFacts; +use sqlbuild_cli::diff_report::types::PhaseSeconds; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::diff_report::support::{ + call, cli_error, error_text, none, object, progress_line, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const NAMESPACE_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target_namespace"; +const RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; +const POLYGLOT_MODULE: &str = "sqlbuild.compiler.sql_analysis.main.import_polyglot_sql"; +const COMMAND_MODELS_MODULE: &str = "sqlbuild.cli.commands.models"; +const LIFECYCLE_MODULE: &str = "sqlbuild.executor.diff.classes.query_artifact_lifecycle"; +const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; +const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +const EXECUTE_QUERY_MODULE: &str = "sqlbuild.executor.diff.main._execute_query"; +const DIFF_MODELS_MODULE: &str = "sqlbuild.executor.diff.models"; +const DIFF_CONFIG_MODULE: &str = "sqlbuild.executor.diff.main._config"; +const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; + +fn failure(py: Python<'_>, message: String, code: &str, help: Option<&str>) -> PyErr { + cli_error( + py, + "CliUserError", + &DiffUserError { + message, + code: code.to_owned(), + help: help.map(str::to_owned), + }, + ) +} + +fn user_failure(py: Python<'_>, error: DiffUserError) -> PyErr { + cli_error(py, "CliUserError", &error) +} + +/// The adapter of the effective adapter name, resolved for the project directory. +pub(crate) fn project_adapter<'py>( + discovered: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult<(String, Bound<'py, PyAny>)> { + let py: Python<'py> = discovered.py(); + let name: Bound<'py, PyAny> = call( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", &discovered.getattr("project_config")?), + ("local_config", &discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = call( + &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, + &[("adapter_name", &name), ("project_dir", project_dir)], + )?; + Ok((name.extract()?, adapter)) +} + +fn query_input( + py: Python<'_>, + inline: Bound<'_, PyAny>, + file: Bound<'_, PyAny>, + label: &str, +) -> PyResult { + if !inline.is_none() { + return inline.extract(); + } + if file.is_none() { + return Err(failure( + py, + format!("raw-query diff requires a {label} query"), + "C229", + None, + )); + } + if !file.call_method0("exists")?.is_truthy()? || !file.call_method0("is_file")?.is_truthy()? { + return Err(failure( + py, + format!("{label} query file does not exist: {}", file.str()?), + "C230", + None, + )); + } + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("encoding", "utf-8")?; + match file.call_method("read_text", (), Some(&keywords)) { + Ok(text) => text.extract(), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + let replacement: PyErr = failure( + py, + format!( + "{label} query file could not be read as UTF-8: {}", + file.str()? + ), + "C231", + None, + ); + replacement.set_cause(py, Some(error)); + Err(replacement) + } + Err(error) => Err(error), + } +} + +fn node_key(node: &Bound<'_, PyAny>) -> PyResult { + match node.getattr_opt("key")? { + Some(key) => Ok(key.str()?.to_string().to_lowercase()), + None => Ok(String::new()), + } +} + +fn read_only_query( + py: Python<'_>, + sql: &str, + label: &str, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let mut normalized: &str = sql.trim(); + if let Some(stripped) = normalized.strip_suffix(';') { + normalized = stripped.trim_end(); + } + if normalized.is_empty() { + return Err(failure( + py, + format!("{label} query must not be empty"), + "C232", + None, + )); + } + let polyglot: Bound<'_, PyAny> = + imported(py, POLYGLOT_MODULE, "import_polyglot_sql")?.call0()?; + if polyglot.is_none() { + return Err(failure( + py, + "raw-query diff requires the bundled SQL parser".to_owned(), + "C233", + None, + )); + } + let dialect: Bound<'_, PyAny> = adapter.call_method0("sql_analysis_dialect")?; + let dialect: Bound<'_, PyAny> = if dialect.is_truthy()? { + dialect + } else { + object(py, "generic")? + }; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("dialect", dialect)?; + let expression: Bound<'_, PyAny> = + match polyglot.call_method("parse_one", (normalized,), Some(&keywords)) { + Ok(expression) => expression, + Err(error) if error.is_instance_of::(py) => { + let replacement: PyErr = failure( + py, + format!( + "{label} query is not one valid SQL statement: {}", + error_text(py, &error)? + ), + "C234", + None, + ); + replacement.set_cause(py, Some(error)); + return Err(replacement); + } + Err(error) => return Err(error), + }; + let mut node_keys: Vec = Vec::new(); + let root_key: String = node_key(&expression)?; + let read_only_root: bool = + ["except", "intersect", "select", "union", "values"].contains(&root_key.as_str()); + if read_only_root { + for node in expression.call_method0("walk")?.try_iter()? { + node_keys.push(node_key(&node?)?); + } + } + let arguments: Bound<'_, PyAny> = expression.getattr_opt("args")?.unwrap_or_else(|| none(py)); + let writes_or_locks: bool = match arguments.downcast::() { + Ok(arguments) => { + arguments + .get_item("into")? + .is_some_and(|into| !into.is_none()) + || arguments + .get_item("locks")? + .map_or(Ok(false), |locks| locks.is_truthy())? + } + Err(_) => false, + }; + read_only_query_problem( + label, + &QueryShapeFacts { + root_key, + node_keys, + writes_or_locks, + }, + ) + .map_err(|error| user_failure(py, error))?; + Ok(normalized.to_owned()) +} + +/// Resolve raw SQL, the active target namespace, the adapter and its connection. +pub(crate) fn prepare_query_diff<'py>( + request: &Bound<'py, PyAny>, + (project_dir, discovered): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let (adapter_name, adapter) = project_adapter(discovered, project_dir)?; + let selected_target: Bound<'py, PyAny> = request.getattr("selected_target")?; + let cli_vars: Bound<'py, PyAny> = request.getattr("cli_vars")?; + let config: Bound<'py, PyAny> = call( + &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, + &[ + ("discovered_inputs", discovered), + ("project_dir", project_dir), + ("selected_target", &selected_target), + ("cli_vars", &cli_vars), + ], + )?; + let database: Bound<'py, PyAny> = if NAMESPACELESS_ADAPTERS.contains(&adapter_name.as_str()) { + none(py) + } else { + config.call_method1("get", ("database",))? + }; + let namespace: Bound<'py, PyAny> = call( + &imported(py, NAMESPACE_MODULE, "build_effective_target_namespace")?, + &[ + ("discovered_inputs", discovered), + ("selected_target", &selected_target), + ("connection_database", &database), + ( + "connection_schema", + &config.call_method1("get", ("schema",))?, + ), + ( + "default_database", + &adapter.call_method0("default_database")?, + ), + ("default_schema", &adapter.call_method0("default_schema")?), + ], + )?; + let (target_name, _target, database, schema): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = namespace.extract()?; + if schema.is_none() { + return Err(failure( + py, + "raw-query diff requires an active target schema".to_owned(), + "C227", + Some("configure the selected target schema before running the comparison"), + )); + } + let left_sql: String = query_input( + py, + request.getattr("left_query")?, + request.getattr("left_query_file")?, + "left", + )?; + let right_sql: String = query_input( + py, + request.getattr("right_query")?, + request.getattr("right_query_file")?, + "right", + )?; + let left_sql: String = read_only_query(py, &left_sql, "left", &adapter)?; + let right_sql: String = read_only_query(py, &right_sql, "right", &adapter)?; + let runtime: Bound<'py, PyTuple> = call( + &imported(py, RUNTIME_MODULE, "build_effective_runtime_config")?, + &[ + ("discovered_inputs", discovered), + ("selected_target", &selected_target), + ("cli_vars", &cli_vars), + ], + )? + .downcast_into()?; + let label = |name: &str, default: &str| -> PyResult { + let value: Bound<'py, PyAny> = request.getattr(name)?; + let text: String = if value.is_truthy()? { + value.extract()? + } else { + default.to_owned() + }; + Ok(text.trim().to_owned()) + }; + let verbose: bool = request.getattr("verbose")?.is_truthy()?; + let examples = |name: &str| -> PyResult> { + object( + py, + effective_max_examples(request.getattr(name)?.extract()?, verbose), + ) + }; + call( + &imported(py, COMMAND_MODELS_MODULE, "QueryDiffPreparation")?, + &[ + ("adapter", &adapter), + ("connection_config", &config), + ("left_sql", &object(py, left_sql)?), + ("right_sql", &object(py, right_sql)?), + ( + "left_label", + &object(py, label("left_label", "left query")?)?, + ), + ( + "right_label", + &object(py, label("right_label", "right query")?)?, + ), + ("selected_target", &target_name), + ("database", &database), + ("schema", &schema), + ("run_id", &runtime.get_item(2)?), + ( + "artifact_ttl", + &discovered + .getattr("project_config")? + .getattr("diff")? + .getattr("query_artifact_ttl")?, + ), + ( + "effective_max_column_examples", + &examples("max_column_examples")?, + ), + ( + "effective_max_row_only_examples", + &examples("max_row_only_examples")?, + ), + ], + ) +} + +fn seconds(started: Instant) -> f64 { + started.elapsed().as_secs_f64() +} + +/// `RowDiffSamplingOverride` and the comparison options a request asks for. +pub(crate) fn execution_options<'py>( + request: &Bound<'py, PyAny>, + extra: &[(&str, &Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = request.py(); + let attribute = |name: &str| request.getattr(name); + let sampling: Bound<'py, PyAny> = call( + &imported(py, DIFF_MODELS_MODULE, "RowDiffSamplingOverride")?, + &[ + ("row_limit", &attribute("sample_rows")?), + ("seed", &attribute("sample_seed")?), + ("exhaustive", &attribute("exhaustive")?), + ], + )?; + let tolerances: Bound<'py, PyAny> = call( + &imported(py, DIFF_CONFIG_MODULE, "parse_cli_tolerance_overrides")?, + &[("values", &attribute("tolerance_overrides")?)], + )?; + let schema_only: Bound<'py, PyAny> = attribute("schema_only")?; + let collect: Bound<'py, PyAny> = object(py, !schema_only.is_truthy()?)?; + let mut pairs: Vec<(&str, &Bound<'py, PyAny>)> = + vec![("schema_only", &schema_only), ("collect_samples", &collect)]; + let columns: Bound<'py, PyAny> = attribute("max_columns")?; + let key: Bound<'py, PyAny> = attribute("unique_key_override")?; + let unkeyed: Bound<'py, PyAny> = attribute("unkeyed")?; + let excluded: Bound<'py, PyAny> = attribute("excluded_columns_override")?; + pairs.extend([ + ("max_columns", &columns), + ("sampling_override", &sampling), + ("unique_key_override", &key), + ("unkeyed", &unkeyed), + ("excluded_columns_override", &excluded), + ("tolerance_overrides", &tolerances), + ]); + pairs.extend_from_slice(extra); + call( + &imported(py, DIFF_MODELS_MODULE, "DiffExecutionOptions")?, + &pairs, + ) +} + +/// Materialize, compare and clean one raw-query pair; the result and per-phase seconds. +pub(crate) fn execute_query_diff<'py>( + request: &Bound<'py, PyAny>, + preparation: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, PhaseSeconds)> { + let py: Python<'py> = request.py(); + let adapter: Bound<'py, PyAny> = preparation.getattr("adapter")?; + let total_started: Instant = Instant::now(); + let mut phases: Vec<(String, f64)> = Vec::new(); + progress_line(py, "Query diff connection START")?; + let connection_started: Instant = Instant::now(); + let connection: Bound<'py, PyAny> = + match adapter.call_method1("connect", (preparation.getattr("connection_config")?,)) { + Ok(connection) => connection, + Err(error) => { + progress_line( + py, + &format!( + "Query diff connection ERROR ({:.2}s)", + seconds(connection_started) + ), + )?; + progress_line( + py, + &format!("Query diff total ERROR ({:.2}s)", seconds(total_started)), + )?; + return Err(error); + } + }; + phases.push(("connection".to_owned(), seconds(connection_started))); + progress_line( + py, + &format!("Query diff connection OK ({:.2}s)", phases[0].1), + )?; + let lifecycle: Bound<'py, PyAny> = + imported(py, LIFECYCLE_MODULE, "QueryDiffArtifactLifecycle")?; + let artifact = |side: &str| -> PyResult> { + call( + &lifecycle.getattr("build")?, + &[ + ("adapter", &adapter), + ("run_id", &preparation.getattr("run_id")?), + ("side", &object(py, side)?), + ("database", &preparation.getattr("database")?), + ("schema", &preparation.getattr("schema")?), + ], + ) + }; + let left: Bound<'py, PyAny> = artifact("left")?; + let right: Bound<'py, PyAny> = artifact("right")?; + let mut created: Vec> = Vec::new(); + let outcome: PyResult> = compare_artifacts( + (request, preparation), + (&connection, &lifecycle), + (&left, &right), + (&mut created, &mut phases), + ); + let cleanup_started: Instant = Instant::now(); + progress_line(py, "Query diff immediate cleanup START")?; + let mut cleanup_errors: Vec = Vec::new(); + for item in created.iter().rev() { + if let Err(error) = call( + &lifecycle.getattr("cleanup")?, + &[ + ("adapter", &adapter), + ("connection", &connection), + ("artifact", item), + ], + ) { + cleanup_errors.push(error); + } + } + if let Err(error) = adapter.call_method1("close", (&connection,)) { + cleanup_errors.push(error); + } + let cleanup_seconds: f64 = seconds(cleanup_started); + if cleanup_errors.is_empty() { + progress_line( + py, + &format!( + "Query diff immediate cleanup OK ({} removed; {cleanup_seconds:.2}s)", + created.len() + ), + )?; + } else { + progress_line( + py, + &format!( + "Query diff immediate cleanup ERROR ({} failed; {cleanup_seconds:.2}s)", + cleanup_errors.len() + ), + )?; + } + phases.push(("cleanup".to_owned(), cleanup_seconds)); + let total: f64 = seconds(total_started); + phases.push(("total".to_owned(), total)); + let status: &str = if outcome.is_err() || !cleanup_errors.is_empty() { + "ERROR" + } else { + "OK" + }; + progress_line(py, &format!("Query diff total {status} ({total:.2}s)"))?; + if !cleanup_errors.is_empty() { + return Err(cleanup_errors.remove(0)); + } + let model: Bound<'py, PyAny> = outcome?; + if model.is_none() { + return Err(cli_error( + py, + "QueryDiffExecutionError", + &DiffUserError { + message: "query diff comparison completed without a result".to_owned(), + code: "C256".to_owned(), + help: None, + }, + )); + } + let result: Bound<'py, PyAny> = call( + &imported(py, DIFF_MODELS_MODULE, "DiffExecutionResult")?, + &[("model_results", &PyTuple::new(py, [model])?.into_any())], + )?; + Ok((result, phases)) +} + +/// Reconcile, inspect, materialize both sides and compare them; the model result. +fn compare_artifacts<'py>( + (request, preparation): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (connection, lifecycle): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (left, right): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (created, phases): (&mut Vec>, &mut Vec<(String, f64)>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let adapter: Bound<'py, PyAny> = preparation.getattr("adapter")?; + phases.push(( + "reconciliation".to_owned(), + reconcile(preparation, connection, lifecycle)?, + )); + let (expires_at, inspection) = inspect(request, preparation, connection, (left, right))?; + phases.push(("inspection".to_owned(), inspection)); + progress_line(py, "Query diff materialization START")?; + let started: Instant = Instant::now(); + for (artifact, sql) in [(left, "left_sql"), (right, "right_sql")] { + if let Err(error) = call( + &lifecycle.getattr("materialize")?, + &[ + ("adapter", &adapter), + ("connection", connection), + ("artifact", artifact), + ("sql", &preparation.getattr(sql)?), + ("expires_at", &expires_at), + ], + ) { + progress_line( + py, + &format!( + "Query diff materialization ERROR ({:.2}s)", + seconds(started) + ), + )?; + return Err(error); + } + created.push(artifact.clone()); + } + let materialization: f64 = seconds(started); + phases.push(("materialization".to_owned(), materialization)); + progress_line( + py, + &format!( + "Query diff materialization OK ({}; expires {}; {materialization:.2}s)", + preparation.getattr("schema")?.str()?, + expires_at.call_method0("isoformat")?.str()? + ), + )?; + progress_line(py, "Query diff comparison START")?; + let started: Instant = Instant::now(); + let name: String = format!( + "{} vs {}", + preparation.getattr("left_label")?.str()?, + preparation.getattr("right_label")?.str()? + ); + let pairs: [(&str, Bound<'py, PyAny>); 3] = [ + ( + "max_column_examples", + preparation.getattr("effective_max_column_examples")?, + ), + ( + "max_row_only_examples", + preparation.getattr("effective_max_row_only_examples")?, + ), + ("comparison_name", object(py, name)?), + ]; + let extra: Vec<(&str, &Bound<'py, PyAny>)> = + pairs.iter().map(|(key, value)| (*key, value)).collect(); + let compared: PyResult> = + execution_options(request, &extra).and_then(|options| { + call( + &imported(py, EXECUTE_QUERY_MODULE, "execute_query_diff")?, + &[ + ("adapter", &adapter), + ("connection", connection), + ("left_relation", &left.getattr("relation")?), + ("right_relation", &right.getattr("relation")?), + ("options", &options), + ], + ) + }); + let comparison: f64 = seconds(started); + phases.push(("comparison".to_owned(), comparison)); + match compared { + Ok(model) => { + progress_line( + py, + &format!("Query diff comparison OK ({comparison:.2}s)"), + )?; + Ok(model) + } + Err(error) => { + progress_line( + py, + &format!("Query diff comparison ERROR ({comparison:.2}s)"), + )?; + Err(error) + } + } +} + +fn reconcile( + preparation: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + lifecycle: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = preparation.py(); + progress_line(py, "Query diff reconciliation START")?; + let started: Instant = Instant::now(); + let now: Bound<'_, PyAny> = imported(py, "datetime", "datetime")? + .call_method1("now", (imported(py, "datetime", "UTC")?,))?; + let cleaned: PyResult> = call( + &lifecycle.getattr("cleanup_expired")?, + &[ + ("adapter", &preparation.getattr("adapter")?), + ("connection", connection), + ("database", &preparation.getattr("database")?), + ("schema", &preparation.getattr("schema")?), + ("now", &now), + ], + ); + let elapsed: f64 = seconds(started); + let result: Bound<'_, PyAny> = match cleaned { + Ok(result) => result, + Err(error) if error.is_instance_of::(py) => { + progress_line( + py, + &format!( + "Query diff reconciliation WARNING recovery cleanup could not inspect historical ownership evidence: {} ({elapsed:.2}s)", + error_text(py, &error)? + ), + )?; + return Ok(elapsed); + } + Err(error) => return Err(error), + }; + progress_line( + py, + &format!( + "Query diff reconciliation OK ({} expired; {elapsed:.2}s)", + result.getattr("cleaned")?.len()? + ), + )?; + for relation in result.getattr("untracked")?.try_iter()? { + progress_line( + py, + &format!( + "Query diff reconciliation WARNING reserved-name relation lacks matching ownership evidence: {}", + relation?.getattr("name")?.str()? + ), + )?; + } + Ok(elapsed) +} + +fn inspect<'py>( + request: &Bound<'py, PyAny>, + preparation: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + artifacts: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyAny>, f64)> { + let py: Python<'py> = request.py(); + progress_line(py, "Query diff inspection START")?; + let started: Instant = Instant::now(); + match inspect_inputs(request, preparation, connection, artifacts) { + Ok(ttl) => { + let elapsed: f64 = seconds(started); + progress_line(py, &format!("Query diff inspection OK ({elapsed:.2}s)"))?; + let now: Bound<'py, PyAny> = imported(py, "datetime", "datetime")? + .call_method1("now", (imported(py, "datetime", "UTC")?,))?; + Ok((ttl.call_method1("add_to", (now,))?, elapsed)) + } + Err(error) => { + progress_line( + py, + &format!("Query diff inspection ERROR ({:.2}s)", seconds(started)), + )?; + Err(error) + } + } +} + +/// Refuse existing artifacts, an invalid TTL and unusable query columns; the parsed TTL. +fn inspect_inputs<'py>( + request: &Bound<'py, PyAny>, + preparation: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + (left, right): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let adapter: Bound<'py, PyAny> = preparation.getattr("adapter")?; + let location = |artifact: &Bound<'py, PyAny>| -> PyResult> { + PyTuple::new( + py, + [ + artifact.getattr("database")?, + artifact.getattr("schema")?, + artifact.getattr("name")?, + ], + ) + }; + let locations: Bound<'py, PyTuple> = PyTuple::new(py, [location(left)?, location(right)?])?; + let lookup: Bound<'py, PyAny> = call( + &imported(py, RELATION_LOOKUP_MODULE, "build_relation_lookup")?, + &[ + ("adapter", &adapter), + ("connection", connection), + ("locations", &locations.into_any()), + ], + )?; + for artifact in [left, right] { + let exists: Bound<'py, PyAny> = call( + &lookup.getattr("exists")?, + &[ + ("database", &artifact.getattr("database")?), + ("schema", &artifact.getattr("schema")?), + ("name", &artifact.getattr("name")?), + ], + )?; + if exists.is_truthy()? { + return Err(failure( + py, + format!( + "query-diff artifact already exists: {}", + artifact.getattr("relation")?.str()? + ), + "C245", + Some("wait for its TTL cleanup or investigate the reserved-name relation"), + )); + } + } + let ttl: Bound<'py, PyAny> = imported(py, CURSOR_MODELS_MODULE, "Duration")? + .call_method1("parse", (preparation.getattr("artifact_ttl")?,))?; + if ttl.is_none() { + return Err(failure( + py, + "diff.query_artifact_ttl must be a positive duration".to_owned(), + "C228", + None, + )); + } + let max_columns: Option = request.getattr("max_columns")?.extract()?; + for (sql, label) in [("left_sql", "left_label"), ("right_sql", "right_label")] { + let columns: Vec = call( + &adapter.getattr("query_column_names")?, + &[ + ("connection", connection), + ("sql", &preparation.getattr(sql)?), + ], + )? + .extract()?; + let label: String = preparation.getattr(label)?.extract()?; + query_column_problem(&label, &columns, max_columns) + .map_err(|error| user_failure(py, error))?; + } + Ok(ttl) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs new file mode 100644 index 0000000000..644245c366 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs @@ -0,0 +1,223 @@ +//! `sqb diff` output: the terminal summary, JSON documents, output publication and exit codes. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, + wrap_pyfunction, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_cli::diff_report::main::render_diff_error_json::render_diff_error_json as error_document; +use sqlbuild_cli::diff_report::main::render_diff_json::render_diff_json; +use sqlbuild_cli::diff_report::main::render_diff_text::render_diff_text; +use sqlbuild_cli::diff_report::models::{ + DiffDocumentOptions, DiffErrorFacts, DiffExampleOptions, DiffModelFacts, DiffTextOptions, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::diff_report::command::{ + execute_prepared_query_diff, run_diff_command, +}; +use crate::bindings::_helpers::diff_report::facts::{diff_facts, json_of}; +use crate::bindings::_helpers::frontier::encoder::emit_error; +use sqlbuild_cli::diff_report::types::PhaseSeconds; + +const DEFAULT_MAX_VALUE_LENGTH: usize = 160; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const TEXT_OPTIONS: (&[&str], &[&str]) = ( + &[ + "result", + "from_label", + "to_label", + "mode_label", + "use_color", + "verbose", + "max_column_examples", + "max_row_only_examples", + ], + &["max_value_length", "suppress_example_values", "outcome"], +); +const ERROR_OPTIONS: (&[&str], &[&str]) = + (&["status", "code", "message"], &["help_text", "size_guard"]); + +fn optional_string(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +fn phase_seconds(value: Bound<'_, PyAny>) -> PyResult>> { + if value.is_none() { + return Ok(None); + } + let mapping: Bound<'_, PyDict> = value.downcast_into()?; + let mut phases: Vec<(String, f64)> = Vec::new(); + for (phase, seconds) in mapping.iter() { + phases.push((phase.extract()?, seconds.extract()?)); + } + Ok(Some(phases)) +} + +/// The example options of explicit `max_value_length`/`suppress_example_values` arguments. +fn example_options(arguments: &KeywordArguments<'_>) -> PyResult { + Ok(DiffExampleOptions { + max_value_length: if arguments.passed("max_value_length")? { + { + let limit: Bound<'_, PyAny> = arguments.value("max_value_length")?; + if limit.is_none() { + None + } else { + Some(limit.extract()?) + } + } + } else { + Some(DEFAULT_MAX_VALUE_LENGTH) + }, + suppress_values: arguments.flag("suppress_example_values")?, + }) +} + +fn document_options( + arguments: &KeywordArguments<'_>, + examples: DiffExampleOptions, +) -> PyResult { + Ok(DiffDocumentOptions { + from_label: arguments.value("from_label")?.extract()?, + to_label: arguments.value("to_label")?.extract()?, + examples, + outcome: optional_string(arguments.value("outcome")?)?, + phase_seconds: phase_seconds(arguments.value("phase_seconds")?)?, + }) +} + +fn json_text(models: &[DiffModelFacts], options: &DiffDocumentOptions) -> PyResult { + render_diff_json(models, options).map_err(emit_error) +} + +/// Render the concise terminal summary of a diff result. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_diff_output(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "render_diff_output", options, TEXT_OPTIONS)?; + let models: Vec = diff_facts(&arguments.value("result")?)?; + let document: DiffDocumentOptions = document_options(&arguments, example_options(&arguments)?)?; + Ok(render_diff_text( + &models, + &document, + &DiffTextOptions { + mode_label: arguments.value("mode_label")?.extract()?, + use_color: arguments.flag("use_color")?, + verbose: arguments.flag("verbose")?, + max_column_examples: arguments.value("max_column_examples")?.extract()?, + max_row_only_examples: arguments.value("max_row_only_examples")?.extract()?, + }, + )) +} + +/// Render a machine-readable incomplete or execution-failure outcome. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_diff_error_json(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "render_diff_error_json", options, ERROR_OPTIONS)?; + let size_guard: Bound<'_, PyAny> = arguments.value("size_guard")?; + error_document(&DiffErrorFacts { + status: arguments.value("status")?.extract()?, + code: arguments.value("code")?.extract()?, + message: arguments.value("message")?.extract()?, + help: optional_string(arguments.value("help_text")?)?, + size_guard: if size_guard.is_none() { + None + } else { + Some(json_of(&size_guard)?) + }, + }) + .map_err(emit_error) +} + +fn mode_label(request: &Bound<'_, PyAny>) -> PyResult { + if request.getattr("schema_only")?.is_truthy()? { + return Ok("schema-only".to_owned()); + } + let bounded: Bound<'_, PyAny> = request.getattr("bounded")?; + if bounded.is_truthy()? { + return Ok(format!("bounded {}", bounded.str()?)); + } + Ok("full".to_owned()) +} + +fn write_stdout(py: Python<'_>, text: &str) -> PyResult<()> { + imported(py, "sys", "stdout")?.call_method1("write", (format!("{text}\n"),))?; + Ok(()) +} + +/// Write the requested JSON file, then print the JSON document or the terminal summary. +pub(crate) fn publish_diff_output( + request: &Bound<'_, PyAny>, + (from_label, to_label): (&str, &str), + (max_column_examples, max_row_only_examples): (i64, i64), + (models, phase_seconds, outcome): (&[DiffModelFacts], Option, Option), +) -> PyResult<()> { + let py: Python<'_> = request.py(); + let max_value_length: Bound<'_, PyAny> = request.getattr("max_value_length")?; + let examples: DiffExampleOptions = DiffExampleOptions { + max_value_length: if request.getattr("full_example_values")?.is_truthy()? { + None + } else if max_value_length.is_truthy()? { + Some(max_value_length.extract()?) + } else { + Some(DEFAULT_MAX_VALUE_LENGTH) + }, + suppress_values: request.getattr("suppress_example_values")?.is_truthy()?, + }; + let document: DiffDocumentOptions = DiffDocumentOptions { + from_label: from_label.to_owned(), + to_label: to_label.to_owned(), + examples, + outcome, + phase_seconds, + }; + let path: Bound<'_, PyAny> = request.getattr("json_output_path")?; + if !path.is_none() { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("parents", true)?; + keywords.set_item("exist_ok", true)?; + path.getattr("parent")? + .call_method("mkdir", (), Some(&keywords))?; + let encoding: Bound<'_, PyDict> = PyDict::new(py); + encoding.set_item("encoding", "utf-8")?; + path.call_method( + "write_text", + (format!("{}\n", json_text(models, &document)?),), + Some(&encoding), + )?; + } + if request.getattr("json_output")?.is_truthy()? { + return write_stdout(py, &json_text(models, &document)?); + } + let use_color: bool = !request.getattr("no_color")?.is_truthy()? + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?; + write_stdout( + py, + &render_diff_text( + models, + &document, + &DiffTextOptions { + mode_label: mode_label(request)?, + use_color, + verbose: request.getattr("verbose")?.is_truthy()?, + max_column_examples: usize::try_from(max_column_examples).unwrap_or_default(), + max_row_only_examples: usize::try_from(max_row_only_examples).unwrap_or_default(), + }, + ), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_diff_output, module)?)?; + module.add_function(wrap_pyfunction!(render_diff_error_json, module)?)?; + module.add_function(wrap_pyfunction!(run_diff_command, module)?)?; + module.add_function(wrap_pyfunction!(execute_prepared_query_diff, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs new file mode 100644 index 0000000000..77f7cfe9b4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs @@ -0,0 +1,160 @@ +//! Python calls, errors and request facts shared by the `sqb diff` command phases. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_cli::diff_report::errors::DiffUserError; +use sqlbuild_cli::diff_report::models::DiffRequestFacts; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; + +/// `name(**pairs)` with keyword arguments. +pub(crate) fn call<'py>( + callable: &Bound<'py, PyAny>, + pairs: &[(&str, &Bound<'py, PyAny>)], +) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(callable.py()); + for (name, value) in pairs { + keywords.set_item(*name, *value)?; + } + callable.call((), Some(&keywords)) +} + +pub(crate) fn none(py: Python<'_>) -> Bound<'_, PyAny> { + py.None().into_bound(py) +} + +pub(crate) fn object<'py, T: pyo3::IntoPyObjectExt<'py>>( + py: Python<'py>, + value: T, +) -> PyResult> { + value.into_bound_py_any(py) +} + +/// `class(message, code=code, help=help)` from the CLI exceptions module. +pub(crate) fn cli_error(py: Python<'_>, class: &str, error: &DiffUserError) -> PyErr { + let raised = (|| -> PyResult> { + let help: Bound<'_, PyAny> = object(py, error.help.as_deref())?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", error.code.as_str())?; + keywords.set_item("help", help)?; + imported(py, EXCEPTIONS_MODULE, class)?.call((error.message.as_str(),), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// The `message`, `code` and `help` of a raised `CliUserError`. +pub(crate) fn user_error_parts(error: &Bound<'_, PyAny>) -> PyResult { + let help: Bound<'_, PyAny> = error.getattr("help")?; + Ok(DiffUserError { + message: error.getattr("message")?.extract()?, + code: error.getattr("code")?.str()?.extract()?, + help: if help.is_none() { + None + } else { + Some(help.extract()?) + }, + }) +} + +/// `error` re-raised as `class` with `error` as its cause. +pub(crate) fn caused(py: Python<'_>, replacement: PyErr, cause: PyErr) -> PyErr { + replacement.set_cause(py, Some(cause)); + replacement +} + +pub(crate) fn matches_class( + py: Python<'_>, + error: &PyErr, + (module, class): (&str, &str), +) -> PyResult { + error.value(py).is_instance(&imported(py, module, class)?) +} + +/// Whether `error` is an `Exception` rather than a `BaseException` such as an interrupt. +pub(crate) fn is_exception(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) +} + +pub(crate) fn error_text(py: Python<'_>, error: &PyErr) -> PyResult { + error.value(py).str()?.extract() +} + +/// Write one line to `sys.stderr`, as `print(..., file=sys.stderr)` does. +pub(crate) fn progress_line(py: Python<'_>, line: &str) -> PyResult<()> { + imported(py, "sys", "stderr")?.call_method1("write", (format!("{line}\n"),))?; + Ok(()) +} + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +fn texts(value: Bound<'_, PyAny>) -> PyResult> { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(item?.str()?.extract()?); + } + Ok(items) +} + +/// The facts of one `DiffCommandRequest` that validation and the size guard read. +pub(crate) fn request_facts(request: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = request.py(); + let attribute = |name: &str| request.getattr(name); + let present = |name: &str| -> PyResult { Ok(!attribute(name)?.is_none()) }; + let number = |name: &str| -> PyResult> { attribute(name)?.extract() }; + let flag = |name: &str| -> PyResult { attribute(name)?.is_truthy() }; + let variables: Bound<'_, PyAny> = attribute("cli_vars")?; + let cli_vars_json: Option = if variables.is_truthy()? { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("sort_keys", true)?; + Some( + imported(py, "json", "dumps")? + .call((variables,), Some(&keywords))? + .extract()?, + ) + } else { + None + }; + Ok(DiffRequestFacts { + from_name: optional_text(attribute("from_name")?)?, + to_name: optional_text(attribute("to_name")?)?, + full: flag("full")?, + schema_only: flag("schema_only")?, + bounded: optional_text(attribute("bounded")?)?, + max_column_examples: number("max_column_examples")?, + max_row_only_examples: number("max_row_only_examples")?, + sample_rows: number("sample_rows")?, + sample_seed: number("sample_seed")?, + exhaustive: flag("exhaustive")?, + max_models: number("max_models")?, + max_columns: number("max_columns")?, + max_value_length: number("max_value_length")?, + full_example_values: flag("full_example_values")?, + suppress_example_values: flag("suppress_example_values")?, + select: texts(attribute("select")?)?, + exclude: texts(attribute("exclude")?)?, + selected_target: optional_text(attribute("selected_target")?)?, + left_query: present("left_query")?, + left_query_file: present("left_query_file")?, + right_query: present("right_query")?, + right_query_file: present("right_query_file")?, + left_label: optional_text(attribute("left_label")?)?, + right_label: optional_text(attribute("right_label")?)?, + unique_key_override: texts(attribute("unique_key_override")?)?, + unkeyed: flag("unkeyed")?, + excluded_columns_override: texts(attribute("excluded_columns_override")?)?, + tolerance_overrides: texts(attribute("tolerance_overrides")?)?, + cli_vars_json, + project_dir: optional_text(attribute("project_dir")?)?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs index 79536ecd32..99f698dc69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs @@ -16,7 +16,10 @@ use sqlbuild_core::freshness_text::models::{ FreshnessSourceResult, FreshnessStatus, StateComparison, }; -const FRESHNESS_PACKAGE: &str = "sqlbuild.compiler.source_freshness.main"; +const ADAPTER_OBSERVATION_MODULE: &str = + "sqlbuild.compiler.source_freshness.main.adapter_observation"; +const OBSERVATION_MODULE: &str = "sqlbuild.compiler.source_freshness.main.observation"; +const NATIVE_MODULE: &str = "sqlbuild._native"; const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; @@ -44,8 +47,7 @@ fn freshness_function<'py>( py: Python<'py>, (module, name): (&str, &str), ) -> PyResult> { - py.import(format!("{FRESHNESS_PACKAGE}.{module}"))? - .getattr(name) + py.import(module)?.getattr(name) } fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { @@ -133,7 +135,7 @@ fn record<'py>( normalize.set_item("value_kind", observation.getattr("value_kind")?)?; let data_version: Bound<'py, PyAny> = freshness_function( py, - ("normalization", "normalize_source_freshness_data_version"), + (NATIVE_MODULE, "normalize_source_freshness_data_version"), )? .call((), Some(&normalize))?; let hash: Bound<'py, PyDict> = PyDict::new(py); @@ -141,11 +143,9 @@ fn record<'py>( hash.set_item("strategy", observation.getattr("strategy")?)?; hash.set_item("value_kind", observation.getattr("value_kind")?)?; hash.set_item("data_version", &data_version)?; - let data_version_hash: Bound<'py, PyAny> = freshness_function( - py, - ("data_version_hash", "source_freshness_data_version_hash"), - )? - .call((), Some(&hash))?; + let data_version_hash: Bound<'py, PyAny> = + freshness_function(py, (NATIVE_MODULE, "source_freshness_data_version_hash"))? + .call((), Some(&hash))?; let fields: Bound<'py, PyDict> = PyDict::new(py); fields.set_item("source_name", observation.getattr("source_name")?)?; fields.set_item("target_database", source.getattr("database")?)?; @@ -200,7 +200,7 @@ fn observed_result( age.set_item("data_version", observation.getattr("data_version")?)?; age.set_item("observed_at", observation.getattr("observed_at")?)?; let age_status: Bound<'_, PyAny> = - freshness_function(py, ("age_policy", "evaluate_source_freshness_age_policy"))? + freshness_function(py, (NATIVE_MODULE, "evaluate_source_freshness_age_policy"))? .call((), Some(&age))?; let previous: Option> = previous_record(state, ¤t)?; let comparison: StateComparison = match &previous { @@ -211,12 +211,10 @@ fn observed_result( options.set_item("previous_record", previous)?; options.set_item("current_record", ¤t)?; options.set_item("lag_tolerance", &lag_tolerance)?; - let equivalent: bool = freshness_function( - py, - ("record_equivalence", "source_freshness_records_equivalent"), - )? - .call((), Some(&options))? - .is_truthy()?; + let equivalent: bool = + freshness_function(py, (NATIVE_MODULE, "source_freshness_records_equivalent"))? + .call((), Some(&options))? + .is_truthy()?; if equivalent { StateComparison::Equivalent { same_hash: previous @@ -271,7 +269,10 @@ fn adapter_batch<'py>( options.set_item("observed_at", &observer.observed_at)?; match freshness_function( py, - ("adapter_observation", "observe_adapter_sources_freshness"), + ( + ADAPTER_OBSERVATION_MODULE, + "observe_adapter_sources_freshness", + ), )? .call((), Some(&options)) { @@ -333,8 +334,11 @@ fn configured_observation<'py>( options.set_item("connection", &observer.connection)?; options.set_item("source", source)?; options.set_item("observed_at", &observer.observed_at)?; - match freshness_function(py, ("observation", "observe_configured_source_freshness"))? - .call((), Some(&options)) + match freshness_function( + py, + (OBSERVATION_MODULE, "observe_configured_source_freshness"), + )? + .call((), Some(&options)) { Ok(observation) => Ok(Ok(observation)), Err(error) => Ok(Err(target_result( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 6fe5dbe9fc..eb83130059 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -28,18 +28,16 @@ const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; -const RUNTIME_PACKAGE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime"; +const PLANNING_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.planning"; +const EXECUTION_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.execution"; +const COMPILATION_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.compilation"; +const CONNECTIONS_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.connections"; const COMMAND_NAME: &str = "janitor"; fn imported<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { py.import(module)?.getattr(name) } -/// One of the janitor runtime phases, which reach the adapter and executor. -fn runtime<'py>(py: Python<'py>, (module, name): (&str, &str)) -> PyResult> { - imported(py, &format!("{RUNTIME_PACKAGE}.{module}"), name) -} - fn keywords<'py>( py: Python<'py>, items: &[(&str, &Bound<'py, PyAny>)], @@ -192,7 +190,7 @@ fn clean( let style: CliStyle = CliStyle { use_color: invocation.getattr("use_color")?.extract()?, }; - let planning: Bound<'_, PyAny> = runtime(py, ("planning", "build_janitor_execution_plan"))? + let planning: Bound<'_, PyAny> = imported(py, PLANNING_MODULE, "build_janitor_execution_plan")? .call( (), Some(&keywords( @@ -227,17 +225,18 @@ fn clean( write(py, "Janitor cancelled.\n")?; return Ok(1); } - let result: Bound<'_, PyAny> = runtime(py, ("execution", "execute_janitor_cleanup"))?.call( - (), - Some(&keywords( - py, - &[ - ("compile_context", compile_context), - ("connection_context", connection_context), - ("planning_result", &planning), - ], - )?), - )?; + let result: Bound<'_, PyAny> = imported(py, EXECUTION_MODULE, "execute_janitor_cleanup")? + .call( + (), + Some(&keywords( + py, + &[ + ("compile_context", compile_context), + ("connection_context", connection_context), + ("planning_result", &planning), + ], + )?), + )?; write( py, &janitor_completion_text(&janitor_result_counts(&result)?, style), @@ -272,10 +271,10 @@ fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult = - runtime(py, ("compilation", "compile_janitor_project"))? + imported(py, COMPILATION_MODULE, "compile_janitor_project")? .call((), Some(&keywords(py, &[("invocation", &invocation)])?))?; let connection_context: Bound<'_, PyAny> = - runtime(py, ("connections", "connect_janitor_warehouse"))?.call( + imported(py, CONNECTIONS_MODULE, "connect_janitor_warehouse")?.call( (), Some(&keywords( py, @@ -292,7 +291,7 @@ fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult> = - runtime(py, ("connections", "close_janitor_warehouse")).and_then(|close| { + imported(py, CONNECTIONS_MODULE, "close_janitor_warehouse").and_then(|close| { close.call( (), Some(&keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs new file mode 100644 index 0000000000..8ae19635f2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -0,0 +1,607 @@ +//! `sqb load`: run the selected source loaders and report each load. + +use std::time::Instant; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::load_command::main::load_counts::load_counts; +use sqlbuild_core::load_command::main::load_exit_code::load_exit_code; +use sqlbuild_core::load_command::main::render_empty_load::render_empty_load; +use sqlbuild_core::load_command::main::render_load_footer::render_load_footer; +use sqlbuild_core::load_command::main::render_load_plan::render_load_plan; +use sqlbuild_core::load_command::main::render_load_ready::render_load_ready; +use sqlbuild_core::load_command::models::{LoadCounts, LoadResultText}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::load::load_progress::{ + LoadCallbackRelay, LoadProgressReporter, load_result_text, +}; +use crate::bindings::_helpers::load::load_selection::{ + LoadProject, LoaderDefaults, loader_defaults, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main.effective_target"; +const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; +const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; +const CURSOR_OVERRIDES_MODULE: &str = + "sqlbuild.compiler.planner.main.cursor_window.typed_cursor_overrides"; +const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; +const PYTHON_NODE_MODELS: &str = "sqlbuild.python_nodes.models"; +const PYTHON_NODE_TYPES: &str = "sqlbuild.python_nodes.types"; +const CURSOR_PARSING_MODULE: &str = "sqlbuild.cli.commands._helpers.input.parsing"; +const LOAD_MODELS: &str = "sqlbuild.executor.load.models"; +const LOAD_RUN_MODULE: &str = "sqlbuild.executor.load.main.run"; +const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; +const EXECUTION_COMMAND: &str = "sqb load"; + +/// The discovered project, selected loads and output context of one load. +struct Invocation<'py> { + project_dir: Bound<'py, PyAny>, + discovered: Bound<'py, PyAny>, + selected: Vec>, + references: Vec>, + relation_sources: Vec>, + style: CliStyle, + stream: Bound<'py, PyAny>, +} + +/// The adapter, connection, runtime and providers prepared for execution. +struct Preparation<'py> { + adapter_name: String, + adapter: Bound<'py, PyAny>, + connection_config: Bound<'py, PyAny>, + runtime: (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>), + concurrency: i64, + cursor_overrides: Bound<'py, PyAny>, + provider_session: Bound<'py, PyAny>, + project_relations: Bound<'py, PyAny>, +} + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn write(stream: &Bound<'_, PyAny>, value: &str) -> PyResult<()> { + stream.call_method1("write", (value,))?; + stream.call_method0("flush")?; + Ok(()) +} + +fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult> { + let project_dir: Bound<'py, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, + }; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + let select: Bound<'py, PyAny> = request.getattr("select")?; + let exclude: Bound<'py, PyAny> = request.getattr("exclude")?; + call_with( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", select.clone()), + ("exclude", exclude.clone()), + ("discovered_inputs", discovered.clone()), + ], + )?; + let selected_target: Bound<'py, PyAny> = request.getattr("selected_target")?; + let target_config: Bound<'py, PyAny> = call_with( + &imported(py, EFFECTIVE_TARGET_MODULE, "build_effective_target_config")?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", selected_target.clone()), + ], + )?; + let defaults: LoaderDefaults<'py> = loader_defaults( + &discovered, + ( + &selected_target, + &target_config, + &request.getattr("cli_vars")?, + ), + )?; + let project: LoadProject<'py> = LoadProject::new(&discovered, &target_config)?; + let selected: Vec> = project.selected_entries( + ( + &select.extract::>()?, + &exclude.extract::>()?, + ), + &defaults, + )?; + let references: Vec> = project.reference_entries(&selected, &defaults)?; + let machine_output: bool = request.getattr("json_output")?.extract()?; + let use_color: bool = !request.getattr("no_color")?.extract::()? + && !machine_output + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .extract::()?; + let stream: Bound<'py, PyAny> = + py.import("sys")? + .getattr(if machine_output { "stderr" } else { "stdout" })?; + Ok(Invocation { + project_dir, + discovered, + selected, + references, + relation_sources: project.sources, + style: CliStyle { use_color }, + stream, + }) +} + +fn user_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, ERRORS_MODULE, "CliUserError").and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn target_name<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + let table: Bound<'_, PyAny> = source.getattr("table")?; + if table.is_truthy()? { + Ok(table) + } else { + source.getattr("name") + } +} + +/// Refuse a load whose skipped intermediate loaders have no target relation yet. +fn check_reference_targets( + invocation: &Invocation<'_>, + (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + if invocation.references.is_empty() { + return Ok(()); + } + let py: Python<'_> = adapter.py(); + let selected_names: Vec = invocation + .selected + .iter() + .map(|source| source.getattr("name")?.extract()) + .collect::>()?; + let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (connection_config,))?; + let checked: PyResult<()> = (|| { + let locations: Vec> = invocation + .references + .iter() + .map(|source| { + PyTuple::new( + py, + [ + source.getattr("database")?, + source.getattr("schema")?, + target_name(source)?, + ], + ) + }) + .collect::>()?; + let lookup: Bound<'_, PyAny> = call_with( + &imported(py, RELATION_LOOKUP_MODULE, "build_relation_lookup")?, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("locations", PyTuple::new(py, locations)?.into_any()), + ], + )?; + for source in &invocation.references { + let target: Bound<'_, PyAny> = target_name(source)?; + let exists: bool = call_with( + &lookup.getattr("exists")?, + &[ + ("database", source.getattr("database")?), + ("schema", source.getattr("schema")?), + ("name", target.clone()), + ], + )? + .is_truthy()?; + if !exists { + return Err(user_error( + py, + format!( + "Selected source load requires intermediate loader '{}', but its target \ + relation '{}' does not exist; use +{} to refresh upstream ingress \ + dependencies", + source.getattr("name")?.str()?, + target.str()?, + selected_names.join(", ") + ), + )); + } + } + Ok(()) + })(); + let closed: PyResult> = adapter.call_method1("close", (&connection,)); + checked?; + closed?; + Ok(()) +} + +/// Source relations loaders must not hard-code, or `None` when enforcement is off. +fn project_relations<'py>( + invocation: &Invocation<'py>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + if !invocation + .discovered + .getattr("project_config")? + .getattr("references")? + .getattr("enforce_explicit")? + .is_truthy()? + { + return Ok(py.None().into_bound(py)); + } + let sources: Bound<'py, PyDict> = PyDict::new(py); + for source in invocation + .relation_sources + .iter() + .chain(&invocation.references) + .chain(&invocation.selected) + { + if source.getattr("expression")?.is_none() { + sources.set_item(source.getattr("name")?, source)?; + } + } + let reference_class: Bound<'py, PyAny> = imported(py, PYTHON_NODE_MODELS, "SqlResourceRef")?; + let source_kind: Bound<'py, PyAny> = + imported(py, PYTHON_NODE_TYPES, "SqlResourceRefKind")?.getattr("SOURCE")?; + let render: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "render_source_relation")?; + let relations: Bound<'py, PyDict> = PyDict::new(py); + for (name, source) in sources.iter() { + let key: Bound<'py, PyAny> = call_with( + &reference_class, + &[("kind", source_kind.clone()), ("name", name)], + )?; + let relation: Bound<'py, PyAny> = + call_with(&render, &[("entry", source), ("adapter", adapter.clone())])?; + relations.set_item(key, relation)?; + } + Ok(relations.into_any()) +} + +fn prepare<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: &Bound<'py, PyAny> = &invocation.discovered; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", invocation.project_dir.clone()), + ], + )?; + let connection_config: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, + &[ + ("discovered_inputs", discovered.clone()), + ("project_dir", invocation.project_dir.clone()), + ("selected_target", request.getattr("selected_target")?), + ("cli_vars", request.getattr("cli_vars")?), + ], + )?; + check_reference_targets(invocation, (&adapter, &connection_config))?; + let runtime: (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = call_with( + &imported( + py, + EFFECTIVE_RUNTIME_MODULE, + "build_effective_runtime_config", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", request.getattr("selected_target")?), + ("cli_vars", request.getattr("cli_vars")?), + ], + )? + .extract()?; + let requested: Bound<'py, PyAny> = request.getattr("concurrency")?; + let concurrency: i64 = if requested.is_none() { + call_with( + &imported( + py, + EFFECTIVE_SETTINGS_MODULE, + "build_effective_settings_config", + )?, + &[("discovered_inputs", discovered.clone())], + )? + .getattr("concurrency")? + .extract()? + } else { + requested.extract()? + }; + let typed: Bound<'py, PyAny> = call_with( + &imported(py, CURSOR_OVERRIDES_MODULE, "typed_cursor_overrides")?, + &[("cursor_overrides", request.getattr("cursor_overrides")?)], + )?; + let cursor_overrides: Bound<'py, PyAny> = if typed.is_truthy()? { + typed + } else { + imported(py, PLANNER_MODELS, "CursorOverrides")?.call0()? + }; + let provider_session: Bound<'py, PyAny> = call_with( + &imported(py, PROVIDER_SESSION_MODULE, "build_provider_session")?, + &[("discovered_providers", discovered.getattr("providers")?)], + )?; + Ok(Preparation { + adapter_name: adapter_name.extract()?, + project_relations: project_relations(invocation, &adapter)?, + adapter, + connection_config, + runtime, + concurrency: concurrency.max(1), + cursor_overrides, + provider_session, + }) +} + +fn cursor<'py>( + overrides: &Bound<'py, PyAny>, + (field, parser): (&str, &str), +) -> PyResult> { + imported(overrides.py(), CURSOR_PARSING_MODULE, parser)?.call1((overrides.getattr(field)?,)) +} + +/// The executor's runtime parameters for this load. +fn runtime_params<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, + preparation: &Preparation<'py>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let (target_name, effective_vars, run_id) = &preparation.runtime; + let overrides: &Bound<'py, PyAny> = &preparation.cursor_overrides; + call_with( + &imported(py, LOAD_MODELS, "LoadRuntimeParams")?, + &[ + ("run_id", run_id.clone()), + ( + "runtime_dir", + invocation + .project_dir + .call_method1("__truediv__", ("target",))?, + ), + ("target", target_name.clone()), + ("vars", effective_vars.clone()), + ("is_reload", request.getattr("reload")?), + ( + "start_cursor_ts", + cursor(overrides, ("start_ts", "parse_cursor_timestamp"))?, + ), + ( + "end_cursor_ts", + cursor(overrides, ("end_ts", "parse_cursor_timestamp"))?, + ), + ( + "start_cursor_int", + cursor(overrides, ("start_int", "parse_cursor_integer"))?, + ), + ( + "end_cursor_int", + cursor(overrides, ("end_int", "parse_cursor_integer"))?, + ), + ( + "use_color", + PyBool::new(py, invocation.style.use_color) + .to_owned() + .into_any(), + ), + ( + "providers", + preparation.provider_session.getattr("providers")?, + ), + ("project_relations", preparation.project_relations.clone()), + ], + ) +} + +/// Run the selected loads with progress callbacks; returns the results and elapsed seconds. +fn execute<'py>( + request: &Bound<'py, PyAny>, + invocation: &Invocation<'py>, + preparation: &Preparation<'py>, +) -> PyResult<(Vec>, f64)> { + let py: Python<'py> = request.py(); + let started: Instant = Instant::now(); + let event_writer: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; + let order: Bound<'py, PyDict> = PyDict::new(py); + for (index, source) in invocation.selected.iter().enumerate() { + order.set_item(source.getattr("name")?, index + 1)?; + } + let reporter: Bound<'py, LoadProgressReporter> = + imported(py, NATIVE_MODULE, "LoadProgressReporter")? + .call( + (), + Some(&{ + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("stream", &invocation.stream)?; + options.set_item("use_color", invocation.style.use_color)?; + options.set_item("source_order", order)?; + options.set_item("total_count", invocation.selected.len())?; + options + }), + )? + .downcast_into()?; + let connection: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", text(py, &preparation.adapter_name)), + ("stream", invocation.stream.clone()), + ( + "blank_line_after_complete", + PyBool::new(py, true).to_owned().into_any(), + ), + ( + "use_color", + PyBool::new(py, invocation.style.use_color) + .to_owned() + .into_any(), + ), + ], + )?; + let relay: Bound<'py, LoadCallbackRelay> = Bound::new( + py, + LoadCallbackRelay { + reporter: reporter.clone().unbind(), + event_writer: event_writer.clone().unbind(), + connection: connection.clone().unbind(), + }, + )?; + let executed: PyResult>> = (|| { + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, LOAD_MODELS, "LoadCallbacks")?, + &[ + ("on_load_start", reporter.getattr("on_start")?), + ("on_load_progress", reporter.getattr("on_progress")?), + ("on_load_complete", relay.getattr("on_load_complete")?), + ( + "on_connection_start", + connection.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + relay.getattr("on_connection_complete")?, + ), + ("on_connection_error", relay.getattr("on_connection_error")?), + ], + )?; + call_with( + &imported(py, LOAD_RUN_MODULE, "run_load_pipeline")?, + &[ + ( + "sources", + PyTuple::new(py, &invocation.selected)?.into_any(), + ), + ( + "reference_sources", + PyTuple::new(py, &invocation.references)?.into_any(), + ), + ( + "loader_functions", + invocation.discovered.getattr("loader_functions")?, + ), + ("connection_config", preparation.connection_config.clone()), + ("adapter", preparation.adapter.clone()), + ( + "max_concurrency", + PyInt::new(py, preparation.concurrency).into_any(), + ), + ("runtime", runtime_params(request, invocation, preparation)?), + ("callbacks", callbacks), + ], + )? + .try_iter()? + .collect() + })(); + let closed: PyResult> = event_writer.call_method0("close"); + let results: Vec> = executed?; + closed?; + Ok((results, started.elapsed().as_secs_f64())) +} + +fn write_json(request: &Bound<'_, PyAny>, results: &[Bound<'_, PyAny>]) -> PyResult<()> { + let py: Python<'_> = request.py(); + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_load_execution_json")?, + &[("results", PyTuple::new(py, results)?.into_any())], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(()) +} + +fn loader_node_flag(source: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = source.py(); + Ok(source + .getattr("meta")? + .call_method1("get", (LOADER_NODE_KEY,))? + .is(PyBool::new(py, true).to_owned())) +} + +/// Run one `sqb load` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_load_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let invocation: Invocation<'_> = invocation(py, request)?; + write( + &invocation.stream, + &render_load_ready(invocation.selected.len(), invocation.style), + )?; + if invocation.selected.is_empty() { + write(&invocation.stream, &render_empty_load(invocation.style))?; + write_json(request, &[])?; + return Ok(0); + } + let preparation: Preparation<'_> = prepare(request, &invocation)?; + let outcome: PyResult<(Vec>, f64)> = (|| { + let entries: Vec<(String, bool)> = invocation + .selected + .iter() + .map(|source| { + Ok(( + source.getattr("name")?.extract()?, + loader_node_flag(source)?, + )) + }) + .collect::>()?; + write( + &invocation.stream, + &render_load_plan(&entries, invocation.style), + )?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", &invocation.stream)?; + options.set_item("command", EXECUTION_COMMAND)?; + options.set_item("target", py.None())?; + options.set_item("concurrency", preparation.concurrency)?; + options.set_item("use_color", invocation.style.use_color)?; + imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&options))?; + execute(request, &invocation, &preparation) + })(); + let closed: PyResult> = preparation.provider_session.call_method0("close"); + let (results, elapsed) = outcome?; + closed?; + let texts: Vec = results + .iter() + .map(load_result_text) + .collect::>()?; + let counts: LoadCounts = load_counts(&texts); + write( + &invocation.stream, + &render_load_footer(&texts, (counts, elapsed), invocation.style), + )?; + write_json(request, &results)?; + Ok(load_exit_code(counts)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_load_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs new file mode 100644 index 0000000000..021b9ce147 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs @@ -0,0 +1,428 @@ +//! Live and persistent progress lines of `sqb load`, as executor callbacks. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::load_command::main::render_load_result::render_load_result; +use sqlbuild_core::load_command::main::render_load_spinner_line::render_load_spinner_line; +use sqlbuild_core::load_command::models::{LoadResultText, LoadSpinnerRow, LoadStatus}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const LIFECYCLE_TYPES: &str = "sqlbuild.adapter.contract.types"; +const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; +const SPINNER_TICK_SECONDS: f64 = 0.1; +const SPINNER_JOIN_SECONDS: f64 = 0.2; +const CLEAR_LINE: &str = "\r\u{1b}[K"; +const HIDE_CURSOR: &str = "\u{1b}[?25l"; +const SHOW_CURSOR: &str = "\u{1b}[?25h"; + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +/// One executor `LoadExecutionResult` as the command renders it. +pub(crate) fn load_result_text(result: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = result.py(); + let log_kind: Bound<'_, PyAny> = + imported(py, LIFECYCLE_TYPES, "LifeCycleEventKind")?.getattr("LOG")?; + let mut log_messages: Vec = Vec::new(); + for event in result.getattr("lifecycle_events")?.try_iter()? { + let event: Bound<'_, PyAny> = event?; + if event.getattr("kind")?.eq(&log_kind)? { + log_messages.push(event.getattr("content")?.extract()?); + } + } + let duration: Bound<'_, PyAny> = result.getattr("duration_ms")?; + Ok(LoadResultText { + source_name: result.getattr("source_name")?.extract()?, + loader_name: result.getattr("loader_name")?.str()?.extract()?, + resource_kind: result + .getattr("resource_kind")? + .getattr("value")? + .extract()?, + status: LoadStatus::from_value( + &result + .getattr("status")? + .getattr("value")? + .extract::()?, + ), + rows_loaded: result.getattr("rows_loaded")?.extract()?, + duration_ms: if duration.is_none() { + None + } else { + Some(duration.extract()?) + }, + log_messages, + error_message: optional_text(result.getattr("error_message")?)?, + warning_messages: result.getattr("warning_messages")?.extract()?, + }) +} + +/// The source currently shown on the live row. +#[derive(Clone)] +struct ActiveSource { + name: String, + resource_kind: String, +} + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Default)] +struct ReporterState { + active: Option, + message: String, + frame: usize, + line_active: bool, + cursor_hidden: bool, + stop_event: Option>, + thread: Option>, +} + +/// TTY-aware renderer for `sqb load` execution progress. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct LoadProgressReporter { + stream: Py, + lines: Py, + write_lock: Py, + style: CliStyle, + source_order: HashMap, + total: i64, + is_tty: bool, + projector: Option>, + state: Mutex, +} + +impl LoadProgressReporter { + fn state(&self) -> MutexGuard<'_, ReporterState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn write(&self, py: Python<'_>, text: &str) -> PyResult<()> { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + Ok(()) + } + + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + within(self.write_lock.bind(py), |_| body()) + } + + fn write_spinner_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let (active, message, frame) = { + let mut state = self.state(); + if !state.line_active { + return Ok(()); + } + let Some(active) = state.active.clone() else { + return Ok(()); + }; + let frame: usize = state.frame; + state.frame = (frame + 1) % 10; + (active, state.message.clone(), frame) + }; + let row: LoadSpinnerRow = LoadSpinnerRow { + ordinal: self.source_order[&active.name], + total: self.total, + resource_kind: active.resource_kind, + name: active.name, + message, + }; + self.write(py, &render_load_spinner_line(&row, frame, self.style)) + }) + } + + fn set_cursor(&self, py: Python<'_>, hidden: bool) -> PyResult<()> { + if self.state().cursor_hidden == hidden { + return Ok(()); + } + self.locked(py, || { + self.write(py, if hidden { HIDE_CURSOR } else { SHOW_CURSOR }) + })?; + self.state().cursor_hidden = hidden; + Ok(()) + } + + fn stop_spinner(&self, py: Python<'_>) -> PyResult<()> { + let (event, thread) = { + let mut state = self.state(); + (state.stop_event.take(), state.thread.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + if let Some(thread) = thread { + let thread: &Bound<'_, PyAny> = thread.bind(py); + if thread.call_method0("is_alive")?.is_truthy()? { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("timeout", SPINNER_JOIN_SECONDS)?; + thread.call_method("join", (), Some(&options))?; + } + } + Ok(()) + } + + fn start_spinner(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + slf.get().stop_spinner(py)?; + let event: Bound<'_, PyAny> = imported(py, "threading", "Event")?.call0()?; + let thread: Bound<'_, PyAny> = call_with( + &imported(py, "threading", "Thread")?, + &[ + ("target", slf.getattr("_spin_until_stopped")?), + ("args", PyTuple::new(py, [&event])?.into_any()), + ("daemon", PyBool::new(py, true).to_owned().into_any()), + ], + )?; + { + let mut state = slf.get().state(); + state.stop_event = Some(event.unbind()); + state.thread = Some(thread.clone().unbind()); + } + thread.call_method0("start")?; + Ok(()) + } +} + +#[pymethods] +impl LoadProgressReporter { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "LoadProgressReporter", + options, + (&["stream", "use_color", "source_order", "total_count"], &[]), + )?; + let stream: Bound<'_, PyAny> = arguments.value("stream")?; + let source_order: HashMap = arguments.value("source_order")?.extract()?; + let total_count: i64 = arguments.value("total_count")?.extract()?; + let lines: Bound<'_, PyAny> = + imported(py, LINES_MODULE, "shared_transient_line_coordinator")?.call0()?; + let projector: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + if !projector.is_none() { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("ordinals", source_order.clone())?; + options.set_item("total", total_count)?; + projector.call_method("configure_resources", (), Some(&options))?; + } + Ok(Self { + is_tty: stream.call_method0("isatty")?.is_truthy()?, + stream: stream.unbind(), + write_lock: lines.getattr("lock")?.unbind(), + lines: lines.unbind(), + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + source_order, + total: total_count, + projector: (!projector.is_none()).then(|| projector.unbind()), + state: Mutex::new(ReporterState::default()), + }) + } + + fn on_start(slf: &Bound<'_, Self>, source: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + if reporter.projector.is_some() || !reporter.is_tty { + return Ok(()); + } + let loader_node: bool = source + .getattr("meta")? + .call_method1("get", (LOADER_NODE_KEY,))? + .is(PyBool::new(py, true).to_owned()); + { + let mut state = reporter.state(); + state.active = Some(ActiveSource { + name: source.getattr("name")?.extract()?, + resource_kind: if loader_node { "loader" } else { "source" }.to_owned(), + }); + state.message = String::new(); + } + reporter.set_cursor(py, true)?; + reporter.locked(py, || { + reporter.state().line_active = true; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", reporter.stream.bind(py))?; + options.set_item("owner", slf)?; + reporter + .lines + .bind(py) + .call_method("claim", (), Some(&options))?; + reporter.write_spinner_line(py) + })?; + Self::start_spinner(slf) + } + + fn on_progress( + &self, + py: Python<'_>, + source: &Bound<'_, PyAny>, + message: String, + ) -> PyResult<()> { + if !self.is_tty { + return Ok(()); + } + let name: String = source.getattr("name")?.extract()?; + { + let mut state = self.state(); + match &state.active { + Some(active) if active.name == name => state.message = message, + _ => return Ok(()), + } + } + self.write_spinner_line(py) + } + + fn on_complete(slf: &Bound<'_, Self>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let text: LoadResultText = load_result_text(result)?; + let mut duration_ms: Option = text.duration_ms; + if let Some(projector) = &reporter.projector { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", &text.source_name)?; + options.set_item("resource_id", format!("source:{}", text.source_name))?; + let canonical: Bound<'_, PyAny> = + projector + .bind(py) + .call_method("consume_resource_terminal", (), Some(&options))?; + if canonical.is_none() { + return Ok(()); + } + duration_ms = Some(canonical.extract()?); + } + reporter.stop_spinner(py)?; + if reporter.is_tty { + reporter.locked(py, || { + reporter.state().line_active = false; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("owner", slf)?; + reporter + .lines + .bind(py) + .call_method("release", (), Some(&options))?; + reporter.write(py, CLEAR_LINE) + })?; + reporter.set_cursor(py, false)?; + } + let ordinal: i64 = *reporter + .source_order + .get(&text.source_name) + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(text.source_name.clone()))?; + reporter.write( + py, + &render_load_result( + &text, + (ordinal, reporter.total), + duration_ms, + reporter.style, + ), + ) + } + + /// Erase the live spinner row so a persistent line can take its place. + fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || self.write(py, CLEAR_LINE)) + } + + /// Draw the live spinner row again below persistent output. + fn redraw_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.write_spinner_line(py) + } + + fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.write_spinner_line(py)?; + } + Ok(()) + } +} + +/// Forward load completions to the reporter and event output, and connection timing to its reporter. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct LoadCallbackRelay { + pub(crate) reporter: Py, + pub(crate) event_writer: Py, + pub(crate) connection: Py, +} + +#[pymethods] +impl LoadCallbackRelay { + fn on_load_complete(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + LoadProgressReporter::on_complete(self.reporter.bind(py), result)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("result", result)?; + options.set_item("plan", py.None())?; + options.set_item("command", PyString::new(py, "load"))?; + self.event_writer + .bind(py) + .call_method("write_build_result", (), Some(&options))?; + Ok(()) + } + + fn on_connection_complete( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.connection_call( + py, + "on_connection_complete", + (connection_count, elapsed_seconds), + ) + } + + fn on_connection_error( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.connection_call( + py, + "on_connection_error", + (connection_count, elapsed_seconds), + ) + } +} + +impl LoadCallbackRelay { + fn connection_call( + &self, + py: Python<'_>, + method: &str, + (connection_count, elapsed_seconds): (i64, f64), + ) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("connection_count", connection_count)?; + options.set_item("elapsed_seconds", elapsed_seconds)?; + self.connection + .bind(py) + .call_method(method, (), Some(&options))?; + Ok(()) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs new file mode 100644 index 0000000000..62c40b9cbe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -0,0 +1,400 @@ +//! Resolve which loaders and sources `sqb load` runs, as `SourceEntry` objects. + +use std::collections::HashMap; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::load_command::errors::UnknownLoadSelector; +use sqlbuild_core::load_command::main::load_reference_loaders::load_reference_loaders; +use sqlbuild_core::load_command::main::select_load_entries::select_load_entries as select_entries; +use sqlbuild_core::load_command::models::{LoadEntry, LoadGraph}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; +const DESTINATION_MODULE: &str = "sqlbuild.spec.contracts.main.loader_destination_parts"; +const CONNECTION_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; +const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; +const LOADER_TABLE_PREFIX: &str = "__loader__"; +const UNKNOWN_SELECTOR_CODE: &str = "C901"; +const UNKNOWN_SELECTOR_HELP: &str = "Use exact managed source names or loader function names."; + +/// The default database and schema of intermediate loader tables. +pub(crate) struct LoaderDefaults<'py> { + pub(crate) database: Bound<'py, PyAny>, + pub(crate) schema: Bound<'py, PyAny>, +} + +/// Discovered loader functions by name and the selection graph over managed sources. +pub(crate) struct LoadProject<'py> { + pub(crate) sources: Vec>, + loaders: HashMap>, + graph: LoadGraph, +} + +/// Every declared source with the target's database and schema filled in where unset. +pub(crate) fn environment_sources<'py>( + discovered: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = discovered.py(); + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let mut sources: Vec> = Vec::new(); + for source_file in discovered.getattr("source_files")?.try_iter()? { + for source in source_file?.getattr("source_entries")?.try_iter()? { + let source: Bound<'py, PyAny> = source?; + let changes: Bound<'py, PyDict> = PyDict::new(py); + for field in ["database", "schema"] { + let value: Bound<'py, PyAny> = source.getattr(field)?; + let effective: Bound<'py, PyAny> = if !value.is_none() || target_config.is_none() { + value + } else { + target_config.getattr(field)? + }; + changes.set_item(field, effective)?; + } + sources.push(replace.call((source,), Some(&changes))?); + } + } + Ok(sources) +} + +impl<'py> LoadProject<'py> { + pub(crate) fn new( + discovered: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + ) -> PyResult { + let py: Python<'py> = discovered.py(); + let sources: Vec> = environment_sources(discovered, target_config)?; + let mut graph: LoadGraph = LoadGraph::default(); + for source in &sources { + let loader: Bound<'py, PyAny> = source.getattr("loader")?; + if !loader.is_none() { + graph + .sources + .push((source.getattr("name")?.extract()?, loader.extract()?)); + } + } + let functions: Vec> = discovered + .getattr("loader_functions")? + .try_iter()? + .collect::>()?; + let names_by_function: Bound<'py, PyDict> = PyDict::new(py); + for function in &functions { + names_by_function.set_item(function.getattr("function")?, function.getattr("name")?)?; + } + let mut loaders: HashMap> = HashMap::new(); + for function in functions { + let name: String = function.getattr("name")?.extract()?; + let mut dependencies: Vec = Vec::new(); + for dependency in function.getattr("depends_on")?.try_iter()? { + if let Some(dependency) = names_by_function.get_item(dependency?)? { + dependencies.push(dependency.extract()?); + } + } + graph.loaders.push((name.clone(), dependencies)); + loaders.insert(name, function); + } + Ok(Self { + sources, + loaders, + graph, + }) + } + + /// The `SourceEntry` an intermediate loader writes, under the loader defaults. + fn loader_entry( + &self, + name: &str, + defaults: &LoaderDefaults<'py>, + ) -> PyResult> { + let loader: &Bound<'py, PyAny> = &self.loaders[name]; + let py: Python<'py> = loader.py(); + let destination: Bound<'py, PyAny> = loader.getattr("destination")?; + let destination: Bound<'py, PyAny> = if destination.is_none() { + PyString::new(py, &format!("{LOADER_TABLE_PREFIX}{name}")).into_any() + } else { + destination + }; + let parts: Bound<'py, PyAny> = call_with( + &imported(py, DESTINATION_MODULE, "loader_destination_parts")?, + &[ + ("destination", destination), + ("default_database", defaults.database.clone()), + ("default_schema", defaults.schema.clone()), + ], + )?; + let meta: Bound<'py, PyDict> = PyDict::new(py); + meta.set_item(LOADER_NODE_KEY, true)?; + let name_value: Bound<'py, PyAny> = PyString::new(py, name).into_any(); + call_with( + &imported(py, CONTRACT_MODELS, "SourceEntry")?, + &[ + ("name", name_value.clone()), + ("database", parts.getattr("database")?), + ("schema", parts.getattr("schema")?), + ("table", parts.getattr("table")?), + ("loader", name_value), + ("write_strategy", loader.getattr("write_strategy")?), + ("cursor_column", loader.getattr("cursor_column")?), + ("unique_key", loader.getattr("unique_key")?), + ("contract", loader.getattr("contract")?), + ("meta", meta.into_any()), + ("columns", loader.getattr("columns")?), + ], + ) + } + + /// The selected loader and source entries in execution order. + pub(crate) fn selected_entries( + &self, + (select, exclude): (&[String], &[String]), + defaults: &LoaderDefaults<'py>, + ) -> PyResult>> { + let py: Python<'py> = defaults.database.py(); + let entries: Vec = select_entries(&self.graph, (select, exclude)) + .map_err(|error| unknown_selector(py, &error))?; + let mut managed: HashMap> = HashMap::new(); + for source in &self.sources { + if !source.getattr("loader")?.is_none() { + managed.insert(source.getattr("name")?.extract()?, source); + } + } + entries + .iter() + .map(|entry| match entry { + LoadEntry::Loader(name) => self.loader_entry(name, defaults), + LoadEntry::Source(name) => Ok(managed[name].clone()), + }) + .collect() + } + + /// Unselected upstream loaders the selected entries only reference. + pub(crate) fn reference_entries( + &self, + selected: &[Bound<'py, PyAny>], + defaults: &LoaderDefaults<'py>, + ) -> PyResult>> { + let pairs: Vec<(String, Option)> = selected + .iter() + .map(|source| { + Ok(( + source.getattr("name")?.extract()?, + source.getattr("loader")?.extract()?, + )) + }) + .collect::>()?; + load_reference_loaders(&self.graph, &pairs) + .iter() + .map(|name| self.loader_entry(name, defaults)) + .collect() + } +} + +fn unknown_selector(py: Python<'_>, error: &UnknownLoadSelector) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", UNKNOWN_SELECTOR_CODE)?; + options.set_item("help", UNKNOWN_SELECTOR_HELP)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call( + (format!( + "sqb load selector '{}' does not match any managed source or loader", + error.name + ),), + Some(&options), + ) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `value.strip()` when it is a string with non-blank text, else `None`. +fn non_empty_text<'py>(value: Option>) -> PyResult>> { + let Some(value) = value else { return Ok(None) }; + if !value.is_instance_of::() { + return Ok(None); + } + let stripped: Bound<'py, PyAny> = value.call_method0("strip")?; + Ok(stripped.is_truthy()?.then_some(stripped)) +} + +/// `target_value or fallback`, reading the target field when there is a target. +fn first_truthy<'py>( + target_config: &Bound<'py, PyAny>, + field: &str, + fallback: Option>, +) -> PyResult> { + let py: Python<'py> = target_config.py(); + if !target_config.is_none() { + let value: Bound<'py, PyAny> = target_config.getattr(field)?; + if value.is_truthy()? { + return Ok(value); + } + } + Ok(fallback.unwrap_or_else(|| py.None().into_bound(py))) +} + +/// Intermediate loader defaults: the target namespace, else the connection's (no database on DuckDB). +pub(crate) fn loader_defaults<'py>( + discovered: &Bound<'py, PyAny>, + (selected_target, target_config, cli_vars): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let connection: Bound<'py, PyDict> = call_with( + &imported( + py, + CONNECTION_CONFIG_MODULE, + "build_effective_connection_config", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", selected_target.clone()), + ("cli_vars", cli_vars.clone()), + ], + )? + .downcast_into()?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let duckdb: Bound<'py, PyAny> = + imported(py, ADAPTER_TYPES_MODULE, "BuiltinAdapter")?.getattr("DUCKDB")?; + let connection_database: Option> = if adapter_name.eq(duckdb)? { + None + } else { + non_empty_text(connection.get_item("database")?)? + }; + Ok(LoaderDefaults { + database: first_truthy(target_config, "database", connection_database)?, + schema: first_truthy( + target_config, + "schema", + non_empty_text(connection.get_item("schema")?)?, + )?, + }) +} + +fn names(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Vec::new()); + } + value.extract() +} + +fn passed_defaults<'py>(arguments: &KeywordArguments<'py>) -> PyResult> { + Ok(LoaderDefaults { + database: arguments.value("loader_default_database")?, + schema: arguments.value("loader_default_schema")?, + }) +} + +/// Select source and intermediate loader execution entries for `sqb load`. +#[pyfunction(name = "select_load_entries")] +#[pyo3(signature = (**options))] +fn select_load_entries_entry<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "select_load_entries", + options, + ( + &["discovered_inputs", "select", "exclude", "target_config"], + &["loader_default_database", "loader_default_schema"], + ), + )?; + let project: LoadProject<'py> = LoadProject::new( + &arguments.value("discovered_inputs")?, + &arguments.value("target_config")?, + )?; + let entries: Vec> = project.selected_entries( + ( + &names(&arguments.value("select")?)?, + &names(&arguments.value("exclude")?)?, + ), + &passed_defaults(&arguments)?, + )?; + PyTuple::new(py, entries) +} + +/// Unselected upstream intermediate loader entries the selected entries only reference. +#[pyfunction] +#[pyo3(signature = (**options))] +fn select_load_reference_entries<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "select_load_reference_entries", + options, + ( + &["discovered_inputs", "selected_sources", "target_config"], + &["loader_default_database", "loader_default_schema"], + ), + )?; + let project: LoadProject<'py> = LoadProject::new( + &arguments.value("discovered_inputs")?, + &arguments.value("target_config")?, + )?; + let selected: Vec> = arguments + .value("selected_sources")? + .try_iter()? + .collect::>()?; + PyTuple::new( + py, + project.reference_entries(&selected, &passed_defaults(&arguments)?)?, + ) +} + +/// The default database and schema intermediate loader tables are written to. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_loader_defaults<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_loader_defaults", + options, + ( + &["discovered_inputs"], + &["selected_target", "target_config", "cli_vars"], + ), + )?; + let defaults: LoaderDefaults<'py> = loader_defaults( + &arguments.value("discovered_inputs")?, + ( + &arguments.value("selected_target")?, + &arguments.value("target_config")?, + &arguments.value("cli_vars")?, + ), + )?; + Ok((defaults.database, defaults.schema)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(select_load_entries_entry, module)?)?; + module.add_function(wrap_pyfunction!(select_load_reference_entries, module)?)?; + module.add_function(wrap_pyfunction!(resolve_loader_defaults, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs new file mode 100644 index 0000000000..68a14122c2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod load_command; +pub(crate) mod load_progress; +pub(crate) mod load_selection; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs new file mode 100644 index 0000000000..198a9ed693 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs @@ -0,0 +1,11 @@ +//! Register the native `sqb load` command, its selection entries and progress reporter. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::load::{load_command, load_progress, load_selection}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + load_command::register(module)?; + load_progress::register(module)?; + load_selection::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 04923fcfd4..1ab5b302a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -5,7 +5,9 @@ pub(crate) mod auditing; pub(crate) mod authored_values; pub(crate) mod boundary; pub(crate) mod cache; +pub(crate) mod check; pub(crate) mod cli; +pub(crate) mod clone; pub(crate) mod command_runtime; pub(crate) mod compile_attachment; pub(crate) mod compile_command; @@ -16,7 +18,10 @@ pub(crate) mod contracts; pub(crate) mod cost_report; pub(crate) mod cursor; pub(crate) mod dag_artifact; +pub(crate) mod dbt_interop; pub(crate) mod debug_command; +pub(crate) mod diagnostics; +pub(crate) mod diff_report; pub(crate) mod discovery; pub(crate) mod fingerprints; pub(crate) mod freshness; @@ -25,6 +30,7 @@ pub(crate) mod graph; pub(crate) mod hand_built; pub(crate) mod janitor; pub(crate) mod lineage; +pub(crate) mod load; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod model_config; @@ -38,9 +44,12 @@ pub(crate) mod render; pub(crate) mod result_output; pub(crate) mod rules; pub(crate) mod scopes; +pub(crate) mod seed_command; pub(crate) mod semantic_checks; pub(crate) mod semantic_shapes; +pub(crate) mod sql_test_command; pub(crate) mod sql_test_glue; pub(crate) mod sqltext; +pub(crate) mod state_rows; pub(crate) mod type_system; pub(crate) mod workspace_commands; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs new file mode 100644 index 0000000000..edbec57e0a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs @@ -0,0 +1,143 @@ +//! The built-in rule catalogue returned as the Python `Rule` models. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::models::RuleMetadata; +use sqlbuild_rules::rules::main::builtin_options::builtin_options; +use sqlbuild_rules::rules::main::catalogue::catalogue; +use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; +use sqlbuild_rules::rules::models::{BuiltinOption, BuiltinOptionKind}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const TYPES_MODULE: &str = "sqlbuild.rule_engine.types"; +const FAMILY_SUFFIX_LENGTH: usize = 3; + +fn keywords<'py>( + py: Python<'py>, + fields: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in fields { + arguments.set_item(name, value)?; + } + Ok(arguments) +} + +fn option<'py>( + option_class: &Bound<'py, PyAny>, + declared: BuiltinOption, +) -> PyResult> { + let py: Python<'py> = option_class.py(); + let (factory, default): (&str, Bound<'py, PyAny>) = match declared.kind { + BuiltinOptionKind::Boolean => ("boolean", PyBool::new(py, false).to_owned().into_any()), + BuiltinOptionKind::StringList => ("string_list", PyTuple::empty(py).into_any()), + }; + let arguments = keywords( + py, + vec![ + ("name", PyString::new(py, declared.name).into_any()), + ("default", default), + ( + "description", + PyString::new(py, declared.description).into_any(), + ), + ], + )?; + option_class.call_method(factory, (), Some(&arguments)) +} + +fn rule<'py>( + models: &Bound<'py, PyModule>, + subjects: &Bound<'py, PyAny>, + check: &Bound<'py, PyAny>, + metadata: RuleMetadata, +) -> PyResult> { + let py: Python<'py> = models.py(); + let option_class: Bound<'py, PyAny> = models.getattr("RuleOption")?; + let options: Vec> = builtin_options(&metadata.code) + .into_iter() + .map(|declared| option(&option_class, declared)) + .collect::>()?; + let guidance: Bound<'py, PyAny> = match metadata.guidance { + Some(guidance) => models.getattr("RuleGuidance")?.call( + (), + Some(&keywords( + py, + vec![ + ( + "good_example", + PyString::new(py, &guidance.good_example).into_any(), + ), + ( + "anti_tautology", + PyString::new(py, &guidance.anti_tautology).into_any(), + ), + ( + "mutation_check", + PyString::new(py, &guidance.mutation_check).into_any(), + ), + ], + )?), + )?, + None => py.None().into_bound(py), + }; + let family: &str = &metadata.code[..metadata.code.len().saturating_sub(FAMILY_SUFFIX_LENGTH)]; + let subject: Bound<'py, PyAny> = subjects.getattr(if metadata.project_wide { + "PROJECT" + } else { + "MODEL" + })?; + let text = |value: &str| PyString::new(py, value).into_any(); + let arguments = keywords( + py, + vec![ + ("code", text(&metadata.code)), + ("family", text(family)), + ("slug", text(&metadata.slug)), + ("message", text(&metadata.message)), + ("remediation", text(&metadata.remediation)), + ("check", check.clone()), + ("options", PyTuple::new(py, options)?.into_any()), + ( + "enabled_by_default", + PyBool::new(py, metadata.enabled_by_default) + .to_owned() + .into_any(), + ), + ("subject", subject), + ("guidance", guidance), + ], + )?; + models.getattr("Rule")?.call((), Some(&arguments)) +} + +/// Every built-in rule as a `Rule` whose `check` is the given placeholder. +#[pyfunction] +#[pyo3(signature = (*, check))] +fn builtin_rules<'py>( + py: Python<'py>, + check: Bound<'py, PyAny>, +) -> PyResult>> { + let models: Bound<'py, PyModule> = py.import(MODELS_MODULE)?; + let subjects: Bound<'py, PyAny> = py.import(TYPES_MODULE)?.getattr("RuleSubject")?; + compiler_guard(|| Ok(catalogue()))? + .into_iter() + .map(|metadata| rule(&models, &subjects, &check, metadata)) + .collect() +} + +/// The built-in SQL rule codes `select` enables and `ignore` does not, in catalogue order. +#[pyfunction] +#[pyo3(signature = (*, select, ignore))] +fn selected_sql_rule_codes(select: Vec, ignore: Vec) -> PyResult> { + compiler_guard(|| Ok(selected_sql_codes(&select, &ignore))) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(builtin_rules, module)?)?; + module.add_function(wrap_pyfunction!(selected_sql_rule_codes, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs new file mode 100644 index 0000000000..8f8e0e3485 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -0,0 +1,245 @@ +//! Custom-rule result cache identities, read-set lookup, result merging and the cache file. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_rules::custom_rule_cache::main::custom_rule_cache_text::custom_rule_cache_text; +use sqlbuild_rules::custom_rule_cache::main::lookup_custom_rule_cache::lookup_custom_rule_cache; +use sqlbuild_rules::custom_rule_cache::main::ordered_custom_findings::ordered_custom_findings; +use sqlbuild_rules::custom_rule_cache::main::parse_custom_rule_cache::parse_custom_rule_cache; +use sqlbuild_rules::custom_rule_cache::main::pending_readsets::pending_readsets; +use sqlbuild_rules::custom_rule_cache::main::rule_identity::rule_identity; +use sqlbuild_rules::custom_rule_cache::main::rule_subjects::rule_subjects; +use sqlbuild_rules::custom_rule_cache::main::store_custom_results::store_custom_results; +use sqlbuild_rules::custom_rule_cache::main::validate_custom_findings::validate_custom_findings; +use sqlbuild_rules::custom_rule_cache::models::{ + CustomCacheLookup, CustomEvaluation, CustomRule, ModelSubjectInput, RuleIdentityInput, + SubjectsInput, +}; +use sqlbuild_rules::custom_rule_cache::types::{ReadSet, StoredRules, SubjectResults}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::rules::findings::{FindingClasses, invalid_finding}; + +const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERROR: &str = "RulesError"; + +/// `(code, subject, (read set, digest) or None, findings)` of one evaluated invocation. +type EvaluationPy<'py> = ( + String, + String, + Option<(ReadSet, Option)>, + Vec>, +); + +/// One custom rule's cache identity from its metadata, source fingerprint and settings. +#[pyfunction] +#[pyo3(signature = (**request))] +fn custom_rule_identity(request: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("custom_rule_identity", request)?; + Ok(rule_identity(&RuleIdentityInput { + code_identity: request.get("code_identity")?.extract()?, + runtime_version: request.get("runtime_version")?.extract()?, + code: request.get("code")?.extract()?, + family: request.get("family")?.extract()?, + slug: request.get("slug")?.extract()?, + message: request.get("message")?.extract()?, + remediation: request.get("remediation")?.extract()?, + subject: request.get("subject")?.extract()?, + implementation_fingerprint: request.get("implementation_fingerprint")?.extract()?, + config_json: plain_value(&request.get("config")?)?.to_string(), + dialect: request.get("dialect")?.extract()?, + project_dir: request.get("project_dir")?.extract()?, + })) +} + +fn readset_tuple<'py>(py: Python<'py>, readset: &ReadSet) -> PyResult> { + let keys: Vec> = readset + .iter() + .map(|key| PyTuple::new(py, key)) + .collect::>()?; + PyTuple::new(py, keys) +} + +fn rules_error(py: Python<'_>, message: String) -> PyResult { + let class: Bound<'_, PyAny> = py.import(RULES_ERRORS_MODULE)?.getattr(RULES_ERROR)?; + Ok(pyo3::PyErr::from_value(class.call1((message,))?)) +} + +fn stored_rules(path: &std::path::Path) -> StoredRules { + let Ok(text) = std::fs::read_to_string(path) else { + return StoredRules::new(); + }; + parse_custom_rule_cache(&text) +} + +/// One custom-rule run's subjects, reused and new results, and the cache file it reads and writes. +#[pyclass(module = "sqlbuild._native")] +struct CustomRuleCache { + path: Option, + rules: Vec, + project: SubjectsInput, + stored: StoredRules, + pending: Vec, + results: SubjectResults, + hits: usize, + misses: usize, +} + +#[pymethods] +impl CustomRuleCache { + /// Load the cache at `path` (`None` when caching is off) for `(code, identity, subject)` rules. + #[new] + #[pyo3(signature = (*, path, rules, target_name, models))] + fn new( + path: Option, + rules: Vec<(String, Option, Option)>, + target_name: Option, + models: Vec<(String, String, Option)>, + ) -> Self { + let project = SubjectsInput { + target_name, + models: models + .into_iter() + .map(|(path, name, materialized)| ModelSubjectInput { + path, + name, + materialized, + }) + .collect(), + }; + let mut rules: Vec = rules + .into_iter() + .map(|(code, identity, subject)| CustomRule { + subjects: rule_subjects(subject.as_deref(), &project), + code, + identity, + subject, + }) + .collect(); + rules.sort_by(|left, right| left.code.cmp(&right.code)); + let stored: StoredRules = path.as_deref().map(stored_rules).unwrap_or_default(); + let pending: Vec = pending_readsets(&stored, &rules); + Self { + path, + rules, + project, + stored, + pending, + results: SubjectResults::new(), + hits: 0, + misses: 0, + } + } + + /// Stored read sets whose current digest decides whether their results are reused. + fn pending_readsets<'py>(&self, py: Python<'py>) -> PyResult>> { + self.pending + .iter() + .map(|readset| readset_tuple(py, readset)) + .collect() + } + + /// Reuse results given each pending read set's digest; returns the subjects still to run. + #[pyo3(signature = (*, digests))] + fn plan(&mut self, digests: Vec>) -> BTreeMap>> { + if self.path.is_none() { + self.misses = self.rules.len(); + return self + .rules + .iter() + .map(|rule| (rule.code.clone(), None)) + .collect(); + } + let current: BTreeMap = self + .pending + .iter() + .zip(digests) + .filter_map(|(readset, digest)| Some((readset.clone(), digest?))) + .collect(); + let lookup: CustomCacheLookup = + lookup_custom_rule_cache(&self.stored, &self.rules, ¤t); + self.results.extend(lookup.hits); + self.hits = lookup.hit_count; + self.misses = lookup.miss_count; + lookup.plan + } + + /// Record evaluated results; with caching on, validate them, store them and write the file. + #[pyo3(signature = (*, evaluated))] + fn record(&mut self, py: Python<'_>, evaluated: Vec>) -> PyResult<()> { + let evaluations: Vec = evaluated + .into_iter() + .map(|(code, subject, cacheable, findings)| { + Ok(CustomEvaluation { + code, + subject, + cacheable, + findings: findings.iter().map(plain_value).collect::>()?, + }) + }) + .collect::>()?; + let Some(path) = self.path.clone() else { + self.extend_results(evaluations); + return Ok(()); + }; + validate_custom_findings(&self.rules, &self.project, &evaluations) + .map_err(|error| rules_error(py, error.0).unwrap_or_else(|error| error))?; + self.stored = store_custom_results(&self.stored, &self.rules, &evaluations); + self.extend_results(evaluations); + let text: String = custom_rule_cache_text(&self.stored, &self.rules); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let temporary: PathBuf = path.with_extension(format!( + "tmp-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + std::fs::write(&temporary, text)?; + std::fs::rename(&temporary, &path)?; + Ok(()) + } + + /// Every finding in rule code, then subject order; a malformed payload is a `RulesError`. + fn findings<'py>(&self, py: Python<'py>) -> PyResult>> { + let classes = FindingClasses::new(py)?; + ordered_custom_findings(&self.rules, &self.results) + .iter() + .map(|finding| { + classes + .finding_of_payload(finding)? + .ok_or_else(|| invalid_finding(py)) + }) + .collect() + } + + #[getter] + fn hits(&self) -> usize { + self.hits + } + + #[getter] + fn misses(&self) -> usize { + self.misses + } +} + +impl CustomRuleCache { + fn extend_results(&mut self, evaluations: Vec) { + for evaluation in evaluations { + self.results + .insert((evaluation.code, evaluation.subject), evaluation.findings); + } + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(custom_rule_identity, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs index cb5c126936..30ab858b7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs @@ -2,10 +2,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_rules::configuration::main::load; -use sqlbuild_rules::constants::API_VERSION; -use sqlbuild_rules::models::CatalogueResponse; -use sqlbuild_rules::rules::main::{catalogue, selected_codes}; +use sqlbuild_rules::rules::main::selected_codes; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::types::CompilerDetach; @@ -18,24 +15,6 @@ fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { .map_err(value_error) } -#[pyfunction] -fn load_config_json(project_dir: &str) -> PyResult { - compiler_guard(|| { - load::load_config_json(std::path::Path::new(project_dir)).map_err(value_error) - }) -} - -#[pyfunction] -fn catalogue_json() -> PyResult { - compiler_guard(|| { - serde_json::to_string(&CatalogueResponse { - version: API_VERSION, - rules: catalogue::catalogue(), - }) - .map_err(value_error) - }) -} - #[pyfunction] fn selected_codes_json(request_json: &str) -> PyResult { compiler_guard(|| selected_codes::selected_codes_json(request_json).map_err(value_error)) @@ -43,8 +22,6 @@ fn selected_codes_json(request_json: &str) -> PyResult { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; - module.add_function(wrap_pyfunction!(load_config_json, module)?)?; - module.add_function(wrap_pyfunction!(catalogue_json, module)?)?; module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?; crate::bindings::_helpers::rules::rows::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs new file mode 100644 index 0000000000..93fd0e3775 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs @@ -0,0 +1,105 @@ +//! Rule findings converted between the Python `Finding` model and native rows and payloads. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::Value; +use sqlbuild_rules::models::Fault; + +const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const FINDING_CLASS: &str = "Finding"; +const PATHLIB_MODULE: &str = "pathlib"; +const PATH_CLASS: &str = "Path"; +const ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERROR: &str = "RulesError"; +const INVALID_FINDING: &str = "native rules engine returned an invalid finding"; + +/// The Python `Finding` and `Path` classes, imported once per conversion batch. +pub(crate) struct FindingClasses<'py> { + finding: Bound<'py, PyAny>, + path: Bound<'py, PyAny>, +} + +impl<'py> FindingClasses<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + Ok(Self { + finding: py.import(MODELS_MODULE)?.getattr(FINDING_CLASS)?, + path: py.import(PATHLIB_MODULE)?.getattr(PATH_CLASS)?, + }) + } + + /// One finding from its native fault row, optionally marked fixable. + pub(crate) fn finding_of_fault( + &self, + fault: Fault, + fixable: bool, + ) -> PyResult> { + let py: Python<'py> = self.finding.py(); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("code", fault.code)?; + arguments.set_item("path", self.path.call1((fault.path,))?)?; + arguments.set_item("line", fault.line)?; + arguments.set_item("column", fault.column)?; + arguments.set_item("message", fault.message)?; + arguments.set_item("remediation", fault.remediation)?; + arguments.set_item("unevaluated", fault.unevaluated)?; + if fixable { + arguments.set_item("fixable", true)?; + } + self.finding.call((), Some(&arguments)) + } + + /// One finding from a stored or custom-rule payload, or `None` when the payload is malformed. + pub(crate) fn finding_of_payload(&self, value: &Value) -> PyResult>> { + let Some(payload) = value.as_object() else { + return Ok(None); + }; + let text = |key: &str| payload.get(key).and_then(Value::as_str).map(str::to_owned); + let number = |key: &str| payload.get(key).and_then(Value::as_u64); + let flag = |key: &str| payload.get(key).and_then(Value::as_bool) == Some(true); + let (Some(code), Some(path), Some(line), Some(column), Some(message), Some(remediation)) = ( + text("code"), + text("path"), + number("line"), + number("column"), + text("message"), + text("remediation"), + ) else { + return Ok(None); + }; + let fault = Fault { + unevaluated: flag("unevaluated"), + code, + path, + line, + column, + message, + remediation, + }; + Ok(Some(self.finding_of_fault(fault, flag("fixable"))?)) + } +} + +/// The native fault row of one Python `Finding`. +pub(crate) fn finding_fault(finding: &Bound<'_, PyAny>) -> PyResult { + Ok(Fault { + unevaluated: finding.getattr("unevaluated")?.extract()?, + code: finding.getattr("code")?.extract()?, + path: finding + .getattr("path")? + .call_method0("as_posix")? + .extract()?, + line: finding.getattr("line")?.extract()?, + column: finding.getattr("column")?.extract()?, + message: finding.getattr("message")?.extract()?, + remediation: finding.getattr("remediation")?.extract()?, + }) +} + +/// The `RulesError` a malformed native or custom-rule finding raises. +pub(crate) fn invalid_finding(py: Python<'_>) -> PyErr { + let raised = || -> PyResult { + let class: Bound<'_, PyAny> = py.import(ERRORS_MODULE)?.getattr(RULES_ERROR)?; + Ok(PyErr::from_value(class.call1((INVALID_FINDING,))?)) + }; + raised().unwrap_or_else(|error| error) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 9956d91032..57a21d4b9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -1,9 +1,15 @@ +pub(crate) mod builtin_catalogue; +pub(crate) mod custom_rule_cache; pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; pub(crate) mod evaluation; +pub(crate) mod findings; pub(crate) mod lint; pub(crate) mod registration; pub(crate) mod relation_keys; pub(crate) mod results; pub(crate) mod rows; +pub(crate) mod rule_cache; +pub(crate) mod rules_config; pub(crate) mod skills; +pub(crate) mod sql_test_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index a9de0613e5..f07045325b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ - custom_rule_evidence, dsl_headers, evaluation, lint, results, skills, + builtin_catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, + results, rule_cache, rules_config, skills, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -11,6 +12,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { lint::register(module)?; dsl_headers::register(module)?; custom_rule_evidence::register(module)?; + rule_cache::register(module)?; + custom_rule_cache::register(module)?; + builtin_catalogue::register(module)?; + rules_config::register(module)?; results::register(module)?; skills::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 066080368b..54a905591f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -1,40 +1,44 @@ //! Build and evaluate the built-in rules request natively from compiled-project rows. -use std::collections::HashMap; +use std::collections::HashSet; use std::path::PathBuf; use std::sync::Mutex; -use pyo3::exceptions::{PyOSError, PyValueError}; +use pyo3::exceptions::PyOSError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, }; use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods}; -use pyo3::{FromPyObject, pyclass, pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_rules::engine::main::build_models::build_models; use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows; use sqlbuild_rules::engine::main::finalize_rows::finalize_rows; use sqlbuild_rules::engine::main::project_model_rows::project_model_rows; use sqlbuild_rules::errors::RowsError; use sqlbuild_rules::models::{ - Declaration, DeclarationMember, DirectTestResourceKind, EvaluateRequest, Fault, Model, - ModelRow, ModelRowsContext, RowsEvaluation, RowsRequest, RulesConfig, SqlScenarioFact, - SqlTestCteFact, SqlTestFact, SqlTestMode, SqlTestParameterFact, SqlTestParameterValueFact, - TestedResource, + Declaration, DeclarationMember, EvaluateRequest, Fault, Model, ModelRow, ModelRowsContext, + RowsEvaluation, RowsRequest, RulesConfig, }; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; +use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; use crate::bindings::types::CompilerDetach; const LEFT_ARGUMENT: &str = "left"; const RIGHT_ARGUMENT: &str = "right"; -type FaultRow = (bool, String, String, u64, u64, String, String); -type CteRow = (String, String); -type ModelTestPayloadPy = (Vec, Vec, Vec, bool, bool); -type TestNamesRow = (Vec, Vec, Vec, Vec); -type EvaluationRow = (Vec, Vec, u64, u64, u64, u64, bool); +type EvaluationRow<'py> = ( + Vec>, + Vec, + u64, + u64, + u64, + u64, + bool, +); /// Name, relative path, members, value, value type and rendering of one declaration. #[derive(FromPyObject)] @@ -47,42 +51,6 @@ struct DeclarationRowPy<'py>( Option, ); -#[derive(FromPyObject)] -struct SqlTestRowPy<'py>( - String, - String, - u64, - String, - Option, - String, - TestNamesRow, - Vec<(String, String)>, - Option, - Option>, -); - -#[derive(FromPyObject)] -struct SqlTestCaseRowPy<'py>( - Option, - String, - Option, - Option, - Vec<(String, String, bool)>, - Vec<(String, Bound<'py, PyAny>)>, -); - -#[derive(FromPyObject)] -struct ScenarioRowPy( - String, - String, - String, - Option, - Vec, - Vec, - Vec, - Vec, -); - /// The request's project facts, which models, tests and scenarios are built beside. #[derive(FromPyObject)] #[pyo3(from_item_all)] @@ -94,7 +62,7 @@ struct ProjectRowsPy<'py> { attached_audit_targets: Vec, public_enums: Vec>, public_constants: Vec>, - initial_findings: Vec, + initial_findings: Vec>, types_equal: Bound<'py, PyAny>, } @@ -104,27 +72,50 @@ struct NativeRulesRequest { request: Mutex>, } +#[pymethods] +impl NativeRulesRequest { + /// The SQL test and scenario facts this request carries, as `{"sql_tests", "sql_scenarios"}` JSON. + fn sql_facts_json(&self) -> PyResult { + let guard = self + .request + .lock() + .map_err(|_| value_error("rules request lock is poisoned"))?; + let rows: &RowsRequest = guard + .as_ref() + .ok_or_else(|| value_error("rules request was already evaluated"))?; + serde_json::to_string(&serde_json::json!({ + "sql_tests": rows.request.sql_tests, + "sql_scenarios": rows.request.sql_scenarios, + })) + .map_err(value_error) + } +} + #[pyfunction] fn build_rules_request<'py>( mut project: ProjectRowsPy<'py>, models: (PyRef<'py, NativeCompiledProject>, Vec), - sql_tests: Vec>, - scenarios: Vec, + sql_tests: Vec>, + scenarios: Vec>, ) -> PyResult { compiler_guard(|| { let mut request: EvaluateRequest = serde_json::from_slice(project.header_json.as_bytes()).map_err(value_error)?; request.sql_tests = sql_tests - .into_iter() - .map(sql_test) + .iter() + .map(sql_test_fact) + .collect::>()?; + request.sql_scenarios = scenarios + .iter() + .map(scenario_fact) .collect::>()?; - request.sql_scenarios = scenarios.into_iter().map(scenario).collect(); request.public_enums = declarations(std::mem::take(&mut project.public_enums))?; request.public_constants = declarations(std::mem::take(&mut project.public_constants))?; - request.initial_findings = std::mem::take(&mut project.initial_findings) - .into_iter() - .map(fault) - .collect(); + request.initial_findings = project + .initial_findings + .iter() + .map(finding_fault) + .collect::>()?; let rows: Vec = project_model_rows(models.0.facts(), &models.1).map_err(value_error)?; request.models = built_models(&project, &request, rows)?; @@ -139,10 +130,10 @@ fn build_rules_request<'py>( } #[pyfunction] -fn evaluate_rules_request( - py: Python<'_>, - request: &Bound<'_, NativeRulesRequest>, -) -> PyResult { +fn evaluate_rules_request<'py>( + py: Python<'py>, + request: &Bound<'py, NativeRulesRequest>, +) -> PyResult> { let request: RowsRequest = request .borrow() .request @@ -157,8 +148,13 @@ fn evaluate_rules_request( RowsError::Rules(message) => value_error(message), RowsError::Io(error) => PyOSError::new_err(error.to_string()), })?; + let classes = FindingClasses::new(py)?; Ok(( - evaluation.faults.into_iter().map(fault_row).collect(), + evaluation + .faults + .into_iter() + .map(|fault| classes.finding_of_fault(fault, false)) + .collect::>()?, evaluation.selected_codes, evaluation.evaluated_models, evaluation.cache_hits, @@ -175,14 +171,14 @@ struct FinalizeRowsPy<'py> { project_dir: String, config_json: Bound<'py, PyBytes>, evaluated_codes: Vec, - findings: Vec, + findings: Vec>, } #[pyfunction] -fn finalize_rule_findings_rows( - py: Python<'_>, - request: FinalizeRowsPy<'_>, -) -> PyResult> { +fn finalize_rule_findings_rows<'py>( + py: Python<'py>, + request: FinalizeRowsPy<'py>, +) -> PyResult>> { let config: RulesConfig = serde_json::from_slice(request.config_json.as_bytes()) .map_err(|error| value_error(format!("invalid findings request: {error}")))?; let FinalizeRowsPy { @@ -191,11 +187,26 @@ fn finalize_rule_findings_rows( findings, .. } = request; - let faults: Vec = findings.into_iter().map(fault).collect(); + let mut fixable: HashSet<(String, String, u64, u64, String)> = HashSet::new(); + let mut faults: Vec = Vec::with_capacity(findings.len()); + for finding in &findings { + let fault: Fault = finding_fault(finding)?; + if finding.getattr("fixable")?.extract::()? { + fixable.insert(fault_identity(&fault)); + } + faults.push(fault); + } let finalized: Vec = py .compiler_detach(|| finalize_rows(project_dir, config, &evaluated_codes, faults)) .map_err(value_error)?; - Ok(finalized.into_iter().map(fault_row).collect()) + let classes = FindingClasses::new(py)?; + finalized + .into_iter() + .map(|fault| { + let is_fixable: bool = fixable.contains(&fault_identity(&fault)); + classes.finding_of_fault(fault, is_fixable) + }) + .collect() } /// `left=` and `right=`: the callback is `types_equal`, whose arguments are keyword-only. @@ -265,166 +276,13 @@ fn declaration_member((name, value): (String, Bound<'_, PyAny>)) -> PyResult) -> PyResult { - let SqlTestRowPy( - source_path, - ownership_root, - block_index, - name, - explicit_name, - mode, - (expected_model_names, assertion_names, assertion_target_model_names, target_model_names), - tested_resources, - model_payload, - case, - ) = row; - let (authored_ctes, expected_ctes, assertion_ctes, has_macro_mocks, has_model_query_overrides) = - model_payload.unwrap_or_default(); - let mut fact = SqlTestFact { - source_path, - ownership_root, - block_index, - name, - explicit_name, - parent_name: None, - case_name: None, - case_index: None, - case_fingerprint: None, - parameter_schema: Vec::new(), - parameters: Vec::new(), - mode: test_mode(&mode)?, - expected_model_names, - assertion_names, - assertion_target_model_names, - target_model_names, - tested_resources: tested_resources - .into_iter() - .map(|(kind, name)| { - Ok(TestedResource { - kind: tested_kind(&kind)?, - name, - }) - }) - .collect::>()?, - authored_ctes: ctes(authored_ctes), - expected_ctes: ctes(expected_ctes), - assertion_ctes: ctes(assertion_ctes), - has_macro_mocks, - has_model_query_overrides, - empty_input_only: false, - }; - if let Some(SqlTestCaseRowPy( - parent_name, - case_name, - case_index, - case_fingerprint, - schema, - values, - )) = case - { - let types: HashMap<&str, &str> = schema - .iter() - .map(|(name, value_type, _)| (name.as_str(), value_type.as_str())) - .collect(); - fact.parameters = values - .into_iter() - .map(|(name, value)| { - let value_type: String = types - .get(name.as_str()) - .ok_or_else(|| PyValueError::new_err(format!("unknown test parameter {name}")))? - .to_string(); - Ok(SqlTestParameterValueFact { - name, - value_type, - value: plain_value(&value)?, - }) - }) - .collect::>()?; - fact.parameter_schema = schema - .into_iter() - .map(|(name, value_type, nullable)| SqlTestParameterFact { - name, - value_type, - nullable, - }) - .collect(); - fact.parent_name = parent_name; - fact.case_name = Some(case_name); - fact.case_index = case_index; - fact.case_fingerprint = case_fingerprint; - } - Ok(fact) -} - -fn ctes(rows: Vec) -> Vec { - rows.into_iter() - .map(|(name, sql)| SqlTestCteFact { name, sql }) - .collect() -} - -fn scenario(row: ScenarioRowPy) -> SqlScenarioFact { - let ScenarioRowPy( - source_path, - ownership_root, - name, - description, - expected_model_names, - assertion_names, - assertion_target_model_names, - target_model_names, - ) = row; - SqlScenarioFact { - source_path, - ownership_root, - name, - description, - expected_model_names, - assertion_names, - assertion_target_model_names, - target_model_names, - } -} - -fn test_mode(value: &str) -> PyResult { - match value { - "model" => Ok(SqlTestMode::Model), - "macro" => Ok(SqlTestMode::Macro), - "udf" => Ok(SqlTestMode::Udf), - "table_fn" => Ok(SqlTestMode::TableFn), - _ => Err(value_error(format!("unknown SQL test mode {value}"))), - } -} - -fn tested_kind(value: &str) -> PyResult { - match value { - "macro" => Ok(DirectTestResourceKind::Macro), - "udf" => Ok(DirectTestResourceKind::Udf), - "table_fn" => Ok(DirectTestResourceKind::TableFn), - _ => Err(value_error(format!("unknown tested resource kind {value}"))), - } -} - -fn fault((unevaluated, code, path, line, column, message, remediation): FaultRow) -> Fault { - Fault { - unevaluated, - code, - path, - line, - column, - message, - remediation, - } -} - -fn fault_row(fault: Fault) -> FaultRow { +fn fault_identity(fault: &Fault) -> (String, String, u64, u64, String) { ( - fault.unevaluated, - fault.code, - fault.path, + fault.code.clone(), + fault.path.clone(), fault.line, fault.column, - fault.message, - fault.remediation, + fault.message.clone(), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs new file mode 100644 index 0000000000..6600388df3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -0,0 +1,201 @@ +//! SQL rule result cache keys, stored-entry lookup and the cache file. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use serde_json::{Value, json}; +use sqlbuild_rules::rule_cache::main::file_rule_key::file_rule_key; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::rules::findings::FindingClasses; +use sqlbuild_rules::rule_cache::main::lint_key::lint_key; +use sqlbuild_rules::rule_cache::main::lookup_rule_cache::lookup_rule_cache; +use sqlbuild_rules::rule_cache::main::model_rule_key::model_rule_key; +use sqlbuild_rules::rule_cache::main::parse_rule_cache::parse_rule_cache; +use sqlbuild_rules::rule_cache::main::rule_cache_text::rule_cache_text; +use sqlbuild_rules::rule_cache::models::{CacheLookup, CachedEntry, LintKeyInput, ModelKeyInput}; +use sqlbuild_rules::rule_cache::types::CachedEntries; + +/// Stored entries as `{path: (key, findings JSON)}`. +type StoredEntries = BTreeMap; + +fn invalid(error: serde_json::Error) -> PyErr { + PyValueError::new_err(error.to_string()) +} + +/// The lint settings part of every SQL rule cache key. +#[pyfunction] +#[pyo3(signature = (*, config))] +fn sql_rule_lint_key(config: &Bound<'_, PyAny>) -> PyResult { + let mut relation_keys: Vec<(String, Vec>)> = Vec::new(); + for item in config + .getattr("relation_keys")? + .call_method0("items")? + .try_iter()? + { + relation_keys.push(item?.extract()?); + } + Ok(lint_key(&LintKeyInput { + dialect: config.getattr("dialect")?.extract()?, + max_literal_length: config.getattr("max_literal_length")?.extract()?, + max_ranking_order_by: config.getattr("max_ranking_order_by")?.extract()?, + relation_keys, + allow_suppressions: config.getattr("allow_suppressions")?.extract()?, + })) +} + +/// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. +#[pyfunction] +#[pyo3(signature = (*, lint_key, codes, models))] +fn sql_model_rule_keys( + lint_key: &str, + codes: Vec, + models: Vec<(String, String, String, Option)>, +) -> BTreeMap { + models + .iter() + .map(|(path, authored_sql, query_sql, contract)| { + let model = ModelKeyInput { + authored_sql, + query_sql, + contract: contract.as_deref(), + }; + (path.clone(), model_rule_key(lint_key, &codes, &model)) + }) + .collect() +} + +/// One non-model SQL file's cache key, or `None` when its findings depend on expansion. +#[pyfunction] +#[pyo3(signature = (**request))] +fn sql_file_rule_key(request: Option<&Bound<'_, PyDict>>) -> PyResult> { + let request = Request::new("sql_file_rule_key", request)?; + let lint_key: String = request.get("lint_key")?.extract()?; + let codes: Vec = request.get("codes")?.extract()?; + let relative_path: String = request.get("relative_path")?.extract()?; + let contents: String = request.get("contents")?.extract()?; + let expanded: bool = request.get("expanded")?.extract()?; + Ok(file_rule_key( + &lint_key, + &codes, + (&relative_path, &contents), + expanded, + )) +} + +/// The usable stored entries of the cache file at `path`; unreadable files read as empty. +#[pyfunction] +#[pyo3(signature = (*, path))] +fn load_sql_rule_cache(path: PathBuf) -> PyResult { + let Ok(text) = std::fs::read_to_string(&path) else { + return Ok(StoredEntries::new()); + }; + parse_rule_cache(&text) + .into_iter() + .map(|(key, entry)| { + let findings: String = serde_json::to_string(&entry.findings).map_err(invalid)?; + Ok((key, (entry.identity, findings))) + }) + .collect() +} + +fn cached_entries(entries: &StoredEntries) -> PyResult { + entries + .iter() + .map(|(path, (identity, findings))| { + let findings: Vec = serde_json::from_str(findings).map_err(invalid)?; + let entry = CachedEntry { + identity: identity.clone(), + findings, + }; + Ok((path.clone(), entry)) + }) + .collect() +} + +/// The stored findings of each path whose key matches; undecodable stored findings miss. +#[pyfunction] +#[pyo3(signature = (*, entries, keys))] +fn lookup_sql_rule_cache<'py>( + py: Python<'py>, + entries: StoredEntries, + keys: Vec<(String, String)>, +) -> PyResult>>> { + let lookup: CacheLookup = lookup_rule_cache(&cached_entries(&entries)?, &keys); + let classes = FindingClasses::new(py)?; + let mut hits: BTreeMap>> = BTreeMap::new(); + for (path, findings) in lookup.hits { + let decoded: Option>> = findings + .iter() + .map(|finding| classes.finding_of_payload(finding)) + .collect::>()?; + if let Some(decoded) = decoded { + hits.insert(path, decoded); + } + } + Ok(hits) +} + +fn finding_payload(finding: &Bound<'_, PyAny>) -> PyResult { + let path: String = finding + .getattr("path")? + .call_method0("as_posix")? + .extract()?; + Ok(json!({ + "unevaluated": finding.getattr("unevaluated")?.extract::()?, + "code": finding.getattr("code")?.extract::()?, + "path": path, + "line": finding.getattr("line")?.extract::>()?, + "column": finding.getattr("column")?.extract::>()?, + "message": finding.getattr("message")?.extract::()?, + "remediation": finding.getattr("remediation")?.extract::>()?, + "fixable": finding.getattr("fixable")?.extract::()?, + })) +} + +/// The stored JSON of one path's findings. +#[pyfunction] +#[pyo3(signature = (*, findings))] +fn encode_sql_rule_findings(findings: Vec>) -> PyResult { + let payloads: Vec = findings + .iter() + .map(finding_payload) + .collect::>()?; + serde_json::to_string(&payloads).map_err(invalid) +} + +/// Replace the cache file at `path` atomically with `entries`. +#[pyfunction] +#[pyo3(signature = (*, path, entries))] +fn write_sql_rule_cache(path: PathBuf, entries: &Bound<'_, PyDict>) -> PyResult<()> { + let mut stored = StoredEntries::new(); + for (key, value) in entries.iter() { + stored.insert(key.extract()?, value.extract()?); + } + let text: String = rule_cache_text(&cached_entries(&stored)?); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let temporary: PathBuf = path.with_extension(format!("tmp-{}", std::process::id())); + std::fs::write(&temporary, text)?; + std::fs::rename(&temporary, &path)?; + Ok(()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_rule_lint_key, module)?)?; + module.add_function(wrap_pyfunction!(sql_model_rule_keys, module)?)?; + module.add_function(wrap_pyfunction!(sql_file_rule_key, module)?)?; + module.add_function(wrap_pyfunction!(load_sql_rule_cache, module)?)?; + module.add_function(wrap_pyfunction!(lookup_sql_rule_cache, module)?)?; + module.add_function(wrap_pyfunction!(encode_sql_rule_findings, module)?)?; + module.add_function(wrap_pyfunction!(write_sql_rule_cache, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs new file mode 100644 index 0000000000..436ffbffa2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -0,0 +1,263 @@ +//! Rules configuration loaded natively and returned as the Python configuration models. + +use std::path::PathBuf; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, Py, pyfunction, wrap_pyfunction}; +use serde_json::Value; +use sqlbuild_rules::configuration::main::forbidden_exceptions::forbidden_exceptions; +use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config as load_config; +use sqlbuild_rules::models::RulesConfig; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; + +struct Models<'py> { + module: Bound<'py, PyModule>, +} + +impl<'py> Models<'py> { + fn build( + &self, + class: &str, + fields: &[(&str, Bound<'py, PyAny>)], + ) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(self.module.py()); + for (name, value) in fields { + arguments.set_item(name, value)?; + } + self.module.getattr(class)?.call((), Some(&arguments)) + } +} + +fn strings<'py>(py: Python<'py>, values: &[String]) -> PyResult> { + Ok(PyTuple::new(py, values)?.into_any()) +} + +fn text<'py>(py: Python<'py>, value: &str) -> PyResult> { + value.into_bound_py_any(py) +} + +fn integers<'py>( + py: Python<'py>, + values: &std::collections::BTreeMap, +) -> PyResult> { + values.into_bound_py_any(py) +} + +/// A scalar option as Python reads it from JSON; floats, tables and nulls are dropped. +fn option_scalar<'py>(py: Python<'py>, value: &Value) -> PyResult>> { + match value { + Value::Bool(flag) => Ok(Some(flag.into_bound_py_any(py)?)), + Value::String(item) => Ok(Some(item.into_bound_py_any(py)?)), + Value::Number(number) => match (number.as_i64(), number.as_u64()) { + (Some(integer), _) => Ok(Some(integer.into_bound_py_any(py)?)), + (None, Some(integer)) => Ok(Some(integer.into_bound_py_any(py)?)), + (None, None) => Ok(None), + }, + _ => Ok(None), + } +} + +/// A list option item as JSON decoding yields it. +fn option_item<'py>(py: Python<'py>, value: &Value) -> PyResult> { + match value { + Value::Array(items) => { + let values: Vec> = items + .iter() + .map(|item| option_item(py, item)) + .collect::>()?; + Ok(pyo3::types::PyList::new(py, values)?.into_any()) + } + Value::Object(entries) => { + let table: Bound<'py, PyDict> = PyDict::new(py); + for (key, item) in entries { + table.set_item(key, option_item(py, item)?)?; + } + Ok(table.into_any()) + } + Value::Null => Ok(py.None().into_bound(py)), + Value::Number(number) if number.as_i64().is_none() && number.as_u64().is_none() => { + number.as_f64().unwrap_or(f64::NAN).into_bound_py_any(py) + } + other => Ok(option_scalar(py, other)?.unwrap_or_else(|| py.None().into_bound(py))), + } +} + +fn rule_options<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> { + let result: Bound<'py, PyDict> = PyDict::new(py); + for (code, options) in &config.rule_options { + let values: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in options { + if let Value::Array(items) = value { + let tuple: Vec> = items + .iter() + .map(|item| option_item(py, item)) + .collect::>()?; + values.set_item(name, PyTuple::new(py, tuple)?)?; + } else if let Some(scalar) = option_scalar(py, value)? { + values.set_item(name, scalar)?; + } + } + result.set_item(code, values)?; + } + Ok(result.into_any()) +} + +fn tables<'py, T>( + py: Python<'py>, + items: &[T], + build: impl Fn(&T) -> PyResult>, +) -> PyResult> { + let values: Vec> = items.iter().map(build).collect::>()?; + Ok(PyTuple::new(py, values)?.into_any()) +} + +fn config_object<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> { + let models = Models { + module: py.import(MODELS_MODULE)?, + }; + let threshold_overrides = tables(py, &config.threshold_overrides, |item| { + models.build( + "ThresholdOverride", + &[ + ("paths", strings(py, &item.paths)?), + ("thresholds", integers(py, &item.thresholds)?), + ("reason", text(py, &item.reason)?), + ], + ) + })?; + let rule_exceptions = tables(py, &config.rule_exceptions, |item| { + models.build( + "RuleExemption", + &[ + ("rule", text(py, &item.rule)?), + ("path", text(py, &item.path)?), + ("reason", text(py, &item.reason)?), + ], + ) + })?; + let graph_edge_exceptions = tables(py, &config.graph_edge_exceptions, |item| { + models.build( + "GraphEdgeExclusion", + &[ + ("consumer", text(py, &item.consumer)?), + ("dependency", text(py, &item.dependency)?), + ("reason", text(py, &item.reason)?), + ], + ) + })?; + let rule_ignores = tables(py, &config.rule_ignores, |item| { + models.build( + "RuleIgnore", + &[ + ("rules", strings(py, &item.rules)?), + ("paths", strings(py, &item.paths)?), + ("reason", text(py, &item.reason)?), + ("selectors", strings(py, &item.selectors)?), + ], + ) + })?; + let select_star_allow = tables(py, &config.select_star_allow, |item| { + models.build( + "SelectStarAllow", + &[ + ("paths", strings(py, &item.paths)?), + ("reason", text(py, &item.reason)?), + ], + ) + })?; + let mut layout: Vec<(&str, Bound<'py, PyAny>)> = + vec![("domain_roots", strings(py, &config.layout.domain_roots)?)]; + if !config.layout.levels.is_empty() { + layout.push(("levels", strings(py, &config.layout.levels)?)); + } + models.build( + "RulesConfig", + &[ + ("select", strings(py, &config.select)?), + ("ignore", strings(py, &config.ignore)?), + ("thresholds", integers(py, &config.thresholds)?), + ("threshold_overrides", threshold_overrides), + ("rule_options", rule_options(py, config)?), + ("rule_exceptions", rule_exceptions), + ("graph_edge_exceptions", graph_edge_exceptions), + ("rule_ignores", rule_ignores), + ( + "allow_exceptions", + config.allow_exceptions.into_bound_py_any(py)?, + ), + ("select_star_allow", select_star_allow), + ("domains", strings(py, &config.domains)?), + ( + "approved_source_tokens", + strings(py, &config.approved_source_tokens)?, + ), + ( + "retired_source_tokens", + config.retired_source_tokens.clone().into_bound_py_any(py)?, + ), + ( + "sql_tests", + models.build( + "SqlTestRulesConfig", + &[( + "pipeline_directory", + text(py, &config.sql_tests.pipeline_directory)?, + )], + )?, + ), + ("layout", models.build("LayoutConfig", &layout)?), + ( + "cache", + models.build( + "RulesCacheConfig", + &[("enabled", config.cache.enabled.into_bound_py_any(py)?)], + )?, + ), + ], + ) +} + +fn rules_error(py: Python<'_>, class: &str, message: String, help: Option) -> PyErr { + let raised = || -> PyResult { + let error_class: Bound<'_, PyAny> = py.import(ERRORS_MODULE)?.getattr(class)?; + let error: Bound<'_, PyAny> = match help { + Some(help) => { + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("help", help)?; + error_class.call((message,), Some(&arguments))? + } + None => error_class.call1((message,))?, + }; + Ok(PyErr::from_value(error)) + }; + raised().unwrap_or_else(|error| error) +} + +/// The project's validated rules configuration as `RulesConfig`. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +fn load_rules_config(py: Python<'_>, project_dir: PathBuf) -> PyResult> { + let config: RulesConfig = compiler_guard(|| Ok(load_config(&project_dir)))? + .map_err(|message| rules_error(py, "RulesError", message, None))?; + if let Some(error) = forbidden_exceptions(&config) { + return Err(rules_error( + py, + "RulesConfigError", + error.message, + Some(error.help), + )); + } + Ok(config_object(py, &config)?.unbind()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(load_rules_config, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs new file mode 100644 index 0000000000..899fc77974 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs @@ -0,0 +1,216 @@ +//! Compiled SQL tests and scenarios read into the native rules request's facts. + +use std::collections::HashMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyString; +use sqlbuild_rules::models::{ + DirectTestResourceKind, SqlScenarioFact, SqlTestCteFact, SqlTestFact, SqlTestMode, + SqlTestParameterFact, SqlTestParameterValueFact, TestedResource, +}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compiled_project::records::plain_typed_value; +use crate::bindings::_helpers::compiled_project::values::plain_value; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const MODEL_PAYLOAD_CLASS: &str = "CompiledModelSqlTestPayload"; +const DESCRIPTION_KEY: &str = "description"; + +fn posix(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("as_posix")?.extract() +} + +/// `primary` when it is truthy, otherwise `fallback`, as Python's `or` picks. +fn either<'py>( + primary: Bound<'py, PyAny>, + fallback: impl FnOnce() -> PyResult>, +) -> PyResult> { + if primary.is_truthy()? { + Ok(primary) + } else { + fallback() + } +} + +fn names(owner: &Bound<'_, PyAny>, attribute: &str) -> PyResult> { + owner.getattr(attribute)?.extract() +} + +fn ctes(payload: &Bound<'_, PyAny>, attribute: &str) -> PyResult> { + payload + .getattr(attribute)? + .try_iter()? + .map(|cte| { + let cte = cte?; + Ok(SqlTestCteFact { + name: cte.getattr("name")?.extract()?, + sql: cte.getattr("sql_body")?.extract()?, + }) + }) + .collect() +} + +fn test_mode(value: &str) -> PyResult { + match value { + "model" => Ok(SqlTestMode::Model), + "macro" => Ok(SqlTestMode::Macro), + "udf" => Ok(SqlTestMode::Udf), + "table_fn" => Ok(SqlTestMode::TableFn), + _ => Err(value_error(format!("unknown SQL test mode {value}"))), + } +} + +fn tested_kind(value: &str) -> PyResult { + match value { + "macro" => Ok(DirectTestResourceKind::Macro), + "udf" => Ok(DirectTestResourceKind::Udf), + "table_fn" => Ok(DirectTestResourceKind::TableFn), + _ => Err(value_error(format!("unknown tested resource kind {value}"))), + } +} + +fn tested_resources(test: &Bound<'_, PyAny>) -> PyResult> { + test.getattr("tested_resources")? + .try_iter()? + .map(|resource| { + let resource = resource?; + let kind: String = resource.getattr("kind")?.getattr("value")?.extract()?; + Ok(TestedResource { + kind: tested_kind(&kind)?, + name: resource.getattr("name")?.extract()?, + }) + }) + .collect() +} + +fn with_payload(test: &Bound<'_, PyAny>, mut fact: SqlTestFact) -> PyResult { + let payload: Bound<'_, PyAny> = test.getattr("payload")?; + let model_payload: Bound<'_, PyAny> = test + .py() + .import(COMPILE_MODELS_MODULE)? + .getattr(MODEL_PAYLOAD_CLASS)?; + if !payload.is_instance(&model_payload)? { + return Ok(fact); + } + fact.authored_ctes = ctes(&payload, "authored_ctes")?; + fact.expected_ctes = ctes(&payload, "expected_ctes")?; + fact.assertion_ctes = ctes(&payload, "assertion_ctes")?; + fact.has_macro_mocks = payload.getattr("macro_mocks")?.is_truthy()?; + fact.has_model_query_overrides = payload.getattr("model_query_overrides")?.is_truthy()?; + Ok(fact) +} + +fn with_case(test: &Bound<'_, PyAny>, mut fact: SqlTestFact) -> PyResult { + let case_name: Option = test.getattr("case_name")?.extract()?; + let Some(case_name) = case_name else { + return Ok(fact); + }; + let mut schema: Vec = Vec::new(); + for parameter in test.getattr("parameter_schema")?.try_iter()? { + let parameter = parameter?; + schema.push(SqlTestParameterFact { + name: parameter.getattr("name")?.extract()?, + value_type: parameter + .getattr("value_type")? + .getattr("value")? + .extract()?, + nullable: parameter.getattr("nullable")?.extract()?, + }); + } + let types: HashMap<&str, &str> = schema + .iter() + .map(|parameter| (parameter.name.as_str(), parameter.value_type.as_str())) + .collect(); + let mut parameters: Vec = Vec::new(); + for item in test.getattr("parameter_values")?.try_iter()? { + let (name, value): (String, Bound<'_, PyAny>) = item?.extract()?; + let value_type: String = types + .get(name.as_str()) + .ok_or_else(|| PyValueError::new_err(format!("unknown test parameter {name}")))? + .to_string(); + parameters.push(SqlTestParameterValueFact { + name, + value_type, + value: plain_value(&plain_typed_value(&value)?)?, + }); + } + fact.parameters = parameters; + fact.parameter_schema = schema; + fact.parent_name = test.getattr("parent_name")?.extract()?; + fact.case_name = Some(case_name); + fact.case_index = test.getattr("case_index")?.extract()?; + fact.case_fingerprint = test.getattr("case_fingerprint")?.extract()?; + Ok(fact) +} + +/// One compiled SQL test as the facts the built-in rules read. +pub(crate) fn sql_test_fact(test: &Bound<'_, PyAny>) -> PyResult { + let test_file: Bound<'_, PyAny> = test.getattr("test_file")?; + let source_path: Bound<'_, PyAny> = either(test.getattr("source_path")?, || { + test_file.getattr("relative_path") + })?; + let ownership_root: Bound<'_, PyAny> = either(test.getattr("ownership_root")?, || { + test_file.getattr("ownership_root") + })?; + let block_index: Bound<'_, PyAny> = either(test.getattr("block_index")?, || { + test.getattr("test_block")?.getattr("test_index") + })?; + let mode: String = test.getattr("mode")?.getattr("value")?.extract()?; + let fact = SqlTestFact { + source_path: posix(&source_path)?, + ownership_root: posix(&ownership_root)?, + block_index: block_index.extract()?, + name: test.getattr("name")?.extract()?, + explicit_name: test.getattr("explicit_name")?.extract()?, + parent_name: None, + case_name: None, + case_index: None, + case_fingerprint: None, + parameter_schema: Vec::new(), + parameters: Vec::new(), + mode: test_mode(&mode)?, + expected_model_names: names(test, "expected_model_names")?, + assertion_names: names(test, "assertion_names")?, + assertion_target_model_names: names(test, "assertion_target_model_names")?, + target_model_names: names(test, "target_model_names")?, + tested_resources: tested_resources(test)?, + authored_ctes: Vec::new(), + expected_ctes: Vec::new(), + assertion_ctes: Vec::new(), + has_macro_mocks: false, + has_model_query_overrides: false, + empty_input_only: false, + }; + with_case(test, with_payload(test, fact)?) +} + +/// One compiled SQL scenario as the facts the built-in rules read. +pub(crate) fn scenario_fact(scenario: &Bound<'_, PyAny>) -> PyResult { + let scenario_file: Bound<'_, PyAny> = scenario.getattr("scenario_file")?; + let source_path: Bound<'_, PyAny> = either(scenario.getattr("source_path")?, || { + scenario_file.getattr("relative_path") + })?; + let ownership_root: Bound<'_, PyAny> = either(scenario.getattr("ownership_root")?, || { + scenario_file.getattr("ownership_root") + })?; + let header: Bound<'_, PyAny> = scenario_file + .getattr("header_values")? + .call_method1("get", (DESCRIPTION_KEY,))?; + let description: Option = if header.is_instance_of::() { + Some(header.extract()?) + } else { + None + }; + Ok(SqlScenarioFact { + source_path: posix(&source_path)?, + ownership_root: posix(&ownership_root)?, + name: scenario.getattr("name")?.extract()?, + description, + expected_model_names: names(scenario, "expected_model_names")?, + assertion_names: names(scenario, "assertion_names")?, + assertion_target_model_names: names(scenario, "assertion_target_model_names")?, + target_model_names: names(scenario, "target_model_names")?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs new file mode 100644 index 0000000000..1b087534ea --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -0,0 +1,493 @@ +//! `sqb seed`: compile the selected seeds, load them, and report. + +use std::collections::HashMap; +use std::time::Instant; + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::seed_command::main::render_seed_completion::render_seed_completion; +use sqlbuild_core::seed_command::main::render_seed_ready::render_seed_ready; +use sqlbuild_core::seed_command::main::render_seed_row::render_seed_row; +use sqlbuild_core::seed_command::main::seed_exit_code::seed_exit_code; +use sqlbuild_core::seed_command::models::SeedRow; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main.static_command"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const SEED_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed"; +const SEED_STATE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed_state"; +const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; +const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; +const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; +const SUCCESS_STATUS: &str = "success"; +const FAILED_STATUS: &str = "failed"; +const EXECUTION_COMMAND: &str = "sqb seed"; + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn status_value(result: &Bound<'_, PyAny>) -> PyResult { + result.getattr("status")?.getattr("value")?.extract() +} + +/// Forward positional connection timing to a reporter that takes keywords. +#[pyclass(module = "sqlbuild._native", frozen)] +struct SeedConnectionRelay { + reporter: Py, +} + +#[pymethods] +impl SeedConnectionRelay { + fn on_connection_complete( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.forward( + py, + "on_connection_complete", + (connection_count, elapsed_seconds), + ) + } + + fn on_connection_error( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.forward( + py, + "on_connection_error", + (connection_count, elapsed_seconds), + ) + } +} + +impl SeedConnectionRelay { + fn forward( + &self, + py: Python<'_>, + method: &str, + (connection_count, elapsed_seconds): (i64, f64), + ) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("connection_count", connection_count)?; + options.set_item("elapsed_seconds", elapsed_seconds)?; + self.reporter + .bind(py) + .call_method(method, (), Some(&options))?; + Ok(()) + } +} + +fn connection_relay<'py>( + reporter: &Bound<'py, PyAny>, +) -> PyResult> { + Bound::new( + reporter.py(), + SeedConnectionRelay { + reporter: reporter.clone().unbind(), + }, + ) +} + +/// Write each finished seed's row and its execution event. +#[pyclass(module = "sqlbuild._native", frozen)] +struct SeedCompletionRelay { + stream: Py, + use_color: bool, + ordinals: HashMap, + total: usize, + event_writer: Py, + plan: Py, + projector: Option>, +} + +#[pymethods] +impl SeedCompletionRelay { + fn on_seed_complete(&self, py: Python<'_>, result: Bound<'_, PyAny>) -> PyResult<()> { + let name: String = result.getattr("seed_name")?.extract()?; + let mut result: Bound<'_, PyAny> = result; + if let Some(projector) = &self.projector { + let duration: Bound<'_, PyAny> = call_with( + &projector.bind(py).getattr("consume_resource_terminal")?, + &[ + ("resource_name", text(py, &name)), + ("resource_id", text(py, &format!("seed:{name}"))), + ], + )?; + if !duration.is_none() { + let milliseconds: Bound<'_, PyAny> = + py.import("builtins")?.getattr("int")?.call1((duration,))?; + let changes: Bound<'_, PyDict> = PyDict::new(py); + changes.set_item("duration_ms", milliseconds)?; + result = py + .import("dataclasses")? + .getattr("replace")? + .call((result,), Some(&changes))?; + } + } + let ordinal: usize = *self + .ordinals + .get(&name) + .ok_or_else(|| PyKeyError::new_err(name.clone()))?; + let style: CliStyle = CliStyle { + use_color: self.use_color, + }; + let row: SeedRow = SeedRow { + name, + ordinal, + total: self.total, + succeeded: status_value(&result)? == SUCCESS_STATUS, + duration_ms: result.getattr("duration_ms")?.extract()?, + error_message: result.getattr("error_message")?.extract()?, + error_code: result.getattr("error_code")?.extract()?, + error_help: result.getattr("error_help")?.extract()?, + }; + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + stream.call_method1("write", (render_seed_row(&row, style),))?; + stream.call_method0("flush")?; + call_with( + &self.event_writer.bind(py).getattr("write_build_result")?, + &[ + ("result", result), + ("plan", self.plan.bind(py).clone()), + ("command", text(py, "seed")), + ], + )?; + Ok(()) + } +} + +/// Compile the selected seeds, reading their stored fingerprints when any are selected. +fn compile_seeds<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; + let adapter: Bound<'py, PyAny> = invocation.getattr("adapter")?; + let connection_config: Bound<'py, PyAny> = invocation.getattr("connection_config")?; + let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; + let connection_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", invocation.getattr("adapter_name")?), + ("stream", stream.clone()), + ("use_color", invocation.getattr("use_color")?), + ], + )?; + stream.call_method1("write", ("\n",))?; + stream.call_method0("flush")?; + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.getattr("effective_project_dir")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let options: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &[ + ("selected_target", request.getattr("selected_target")?), + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("connection_config", connection_config.clone()), + ("cli_vars", request.getattr("cli_vars")?), + ("external_sql_reference_resolver", resolver), + ], + )?; + let context: Bound<'py, PyAny> = call_with( + &imported(py, STATIC_COMMAND_MODULE, "compile_static_command_context")?, + &[ + ("discovered_inputs", discovered), + ("adapter", adapter.clone()), + ("options", options), + ], + )?; + let project: Bound<'py, PyAny> = context.getattr("project")?; + let scope: Bound<'py, PyAny> = context.getattr("scope")?; + let selected_keys: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + let mut selected: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed: Bound<'py, PyAny> = seed?; + if selected_keys.contains(seed.getattr("key")?)? { + selected.push(seed); + } + } + let fingerprints: Bound<'py, PyAny> = if selected.is_empty() { + PyDict::new(py).into_any() + } else { + let relay: Bound<'py, SeedConnectionRelay> = connection_relay(&connection_progress)?; + let hooks: Bound<'py, PyAny> = call_with( + &imported(py, RUNTIME_CONTRACT_MODELS, "ConnectionHooks")?, + &[ + ( + "on_connection_start", + connection_progress.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + relay.getattr("on_connection_complete")?, + ), + ("on_connection_error", relay.getattr("on_connection_error")?), + ], + )?; + let connection: Bound<'py, PyAny> = call_with( + &imported(py, OPEN_CONNECTION_MODULE, "open_connection_with_hooks")?, + &[ + ("adapter", adapter.clone()), + ("connection_config", connection_config), + ("hooks", hooks), + ], + )?; + let read: PyResult> = call_with( + &imported(py, SEED_STATE_MODULE, "read_selected_seed_fingerprints")?, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("seeds", PyTuple::new(py, &selected)?.into_any()), + ], + ); + let closed: PyResult> = adapter.call_method1("close", (connection,)); + let read: Bound<'py, PyAny> = read?; + closed?; + read + }; + let plan_output: Bound<'py, PyAny> = call_with( + &imported(py, SEED_PLAN_MODULE, "build_seed_command_plan")?, + &[ + ("project", project), + ("scope", scope), + ("relations", context.getattr("relations")?), + ("fingerprints", fingerprints), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "build_static_pipeline_result")?, + &[("context", context), ("plan_output", plan_output)], + ) +} + +fn seed_names(pipeline: &Bound<'_, PyAny>) -> PyResult> { + pipeline + .getattr("plan_output")? + .getattr("seed_entries")? + .try_iter()? + .map(|entry| entry?.getattr("name")?.extract()) + .collect() +} + +/// Write the ready header, the selected seeds and the execution header. +fn write_header( + invocation: &Bound<'_, PyAny>, + pipeline: &Bound<'_, PyAny>, + concurrency: i64, +) -> PyResult<()> { + let py: Python<'_> = invocation.py(); + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + let use_color: Bound<'_, PyAny> = invocation.getattr("use_color")?; + let style: CliStyle = CliStyle { + use_color: use_color.is_truthy()?, + }; + stream.call_method1("write", (render_seed_ready(&seed_names(pipeline)?, style),))?; + let header: Bound<'_, PyDict> = PyDict::new(py); + header.set_item("stream", &stream)?; + header.set_item("command", EXECUTION_COMMAND)?; + header.set_item("target", py.None())?; + header.set_item("concurrency", concurrency)?; + header.set_item("use_color", use_color)?; + imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&header))?; + Ok(()) +} + +/// Load the planned seeds, writing one row per finished seed. +fn load_seeds<'py>( + invocation: &Bound<'py, PyAny>, + pipeline: &Bound<'py, PyAny>, + concurrency: i64, +) -> PyResult>> { + let py: Python<'py> = invocation.py(); + let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; + let use_color: Bound<'py, PyAny> = invocation.getattr("use_color")?; + let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let project: Bound<'py, PyAny> = pipeline.getattr("project")?; + let names: Vec = seed_names(pipeline)?; + let ordinals: HashMap = names + .iter() + .enumerate() + .map(|(index, name)| (name.clone(), index + 1)) + .collect(); + let event_writer: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; + let projector: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + let projector: Option> = if projector.is_none() { + None + } else { + let order: Bound<'py, PyDict> = PyDict::new(py); + for (index, name) in names.iter().enumerate() { + order.set_item(name, index + 1)?; + } + call_with( + &projector.getattr("configure_resources")?, + &[ + ("ordinals", order.into_any()), + ("total", PyInt::new(py, names.len()).into_any()), + ], + )?; + Some(projector.unbind()) + }; + let completion: Bound<'py, SeedCompletionRelay> = Bound::new( + py, + SeedCompletionRelay { + stream: stream.clone().unbind(), + use_color: use_color.is_truthy()?, + ordinals, + total: names.len(), + event_writer: event_writer.clone().unbind(), + plan: plan.clone().unbind(), + projector, + }, + )?; + let connection_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", invocation.getattr("adapter_name")?), + ("stream", stream), + ( + "blank_line_after_complete", + PyBool::new(py, true).to_owned().into_any(), + ), + ("use_color", use_color), + ], + )?; + let relay: Bound<'py, SeedConnectionRelay> = connection_relay(&connection_progress)?; + let loaded: PyResult>> = (|| { + call_with( + &imported(py, EXECUTOR_RUN_MODULE, "run_seed_pipeline")?, + &[ + ("plan", plan), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("adapter", invocation.getattr("adapter")?), + ("max_concurrency", PyInt::new(py, concurrency).into_any()), + ("run_id", project.getattr("run_id")?), + ( + "query_change_tracking", + project + .getattr("settings")? + .getattr("query_change_tracking")?, + ), + ("on_seed_complete", completion.getattr("on_seed_complete")?), + ( + "on_connection_start", + connection_progress.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + relay.getattr("on_connection_complete")?, + ), + ("on_connection_error", relay.getattr("on_connection_error")?), + ], + )? + .try_iter()? + .collect() + })(); + let closed: PyResult> = event_writer.call_method0("close"); + let results: Vec> = loaded?; + closed?; + Ok(results) +} + +/// Run one `sqb seed` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_seed_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let invocation: Bound<'_, PyAny> = call_with( + &imported(py, INVOCATION_MODULE, "resolve_command_invocation")?, + &[ + ("request", request.clone()), + ( + "invocation_type", + imported(py, COMMAND_MODELS, "SeedInvocation")?, + ), + ], + )?; + let pipeline: Bound<'_, PyAny> = compile_seeds(request, &invocation)?; + let requested: Bound<'_, PyAny> = request.getattr("concurrency")?; + let concurrency: i64 = if requested.is_none() { + pipeline + .getattr("project")? + .getattr("settings")? + .getattr("concurrency")? + .extract::()? + } else { + requested.extract::()? + } + .max(1); + write_header(&invocation, &pipeline, concurrency)?; + let started: Instant = Instant::now(); + let results: Vec> = load_seeds(&invocation, &pipeline, concurrency)?; + let elapsed: f64 = started.elapsed().as_secs_f64(); + let mut passed: usize = 0; + let mut failed: usize = 0; + for result in &results { + match status_value(result)?.as_str() { + SUCCESS_STATUS => passed += 1, + FAILED_STATUS => failed += 1, + _ => {} + } + } + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + let style: CliStyle = CliStyle { + use_color: invocation.getattr("use_color")?.is_truthy()?, + }; + stream.call_method1( + "write", + (render_seed_completion( + (passed, failed, results.len()), + elapsed, + style, + ),), + )?; + stream.call_method0("flush")?; + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_seed_execution_json")?, + &[ + ("results", PyTuple::new(py, &results)?.into_any()), + ("plan", pipeline.getattr("plan_output")?), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(seed_exit_code(failed)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_seed_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/mod.rs new file mode 100644 index 0000000000..2ef3aae6d5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod command; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/registration.rs new file mode 100644 index 0000000000..7fcfbefc33 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/registration.rs @@ -0,0 +1,9 @@ +//! Register the native `sqb seed` command. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::seed_command::command; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs new file mode 100644 index 0000000000..1e38419e09 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod registration; +pub(crate) mod test_command; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs new file mode 100644 index 0000000000..daf61f4830 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs @@ -0,0 +1,9 @@ +//! Register the native `sqb test` command and its progress labels. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::sql_test_command::test_command; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + test_command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs new file mode 100644 index 0000000000..fc9b4685a8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -0,0 +1,755 @@ +//! `sqb test`: compile the selected SQL unit tests, inspect or run them, and report. + +use std::collections::{BTreeSet, HashMap}; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; +use sqlbuild_core::test_command::main::parameterized_test_label::parameterized_test_label; +use sqlbuild_core::test_command::main::render_test_completion::render_test_completion; +use sqlbuild_core::test_command::main::render_test_inspection::render_test_inspection; +use sqlbuild_core::test_command::main::render_test_ready::render_test_ready; +use sqlbuild_core::test_command::main::test_exit_code::test_exit_code; +use sqlbuild_core::test_command::main::test_group_name::test_group_name; +use sqlbuild_core::test_command::main::test_name_width::test_name_width; +use sqlbuild_core::test_command::main::test_outcome_status::test_outcome_status; +use sqlbuild_core::test_command::models::{ + TestFailure, TestInspectionEntry, TestParameter, TestPlanWarning, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::result_output::integration_values::json_value; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main.static_command"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; +const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; +const NESTED_PROGRESS_MODULE: &str = + "sqlbuild.cli.progress.classes.nested_command_progress_callbacks"; +const PROGRESS_MODELS: &str = "sqlbuild.cli.progress.models"; +const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; +const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; +const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; +const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main.resource_id"; +const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_test_runtime_target"; +const DECIMAL_KIND: &str = "decimal"; +const PASS_OUTCOME: &str = "pass"; +const EXPECT_LABEL: &str = "expect"; +const UNKNOWN_GROUP: &str = "(unknown)"; +const EXECUTION_COMMAND: &str = "sqb test"; +const UNKNOWN_CASE_CODE: &str = "S008"; + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn strings(values: Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +/// A payload as `json.dumps(payload, ensure_ascii=True, separators=(",", ":"))`. +fn compact_json(payload: &Bound<'_, PyAny>) -> PyResult { + let Some(value) = json_value(payload)? else { + let options: Bound<'_, PyDict> = PyDict::new(payload.py()); + options.set_item("ensure_ascii", true)?; + options.set_item("separators", (",", ":"))?; + return payload + .py() + .import("json")? + .call_method("dumps", (payload,), Some(&options))? + .extract(); + }; + dumps( + &value, + &JsonDialect::Stdlib(StdlibJsonOptions::new(None).with_separators(",", ":")), + ) + .map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + +/// The label of one test or result: its name, plus source and values for parameterized cases. +fn test_label(item: &Bound<'_, PyAny>, name_field: &str) -> PyResult { + let name: String = item.getattr(name_field)?.extract()?; + let source_path: Bound<'_, PyAny> = item.getattr("source_path")?; + let values: Vec> = item + .getattr("parameter_values")? + .try_iter()? + .collect::>()?; + if source_path.is_none() || values.is_empty() { + return Ok(name); + } + let mut schema: HashMap> = HashMap::new(); + for declaration in item.getattr("parameter_schema")?.try_iter()? { + let declaration: Bound<'_, PyAny> = declaration?; + schema.insert(declaration.getattr("name")?.extract()?, declaration); + } + let mut parameters: Vec = Vec::new(); + for pair in values { + let (value_name, value): (String, Bound<'_, PyAny>) = pair.extract()?; + let declaration: &Bound<'_, PyAny> = schema + .get(&value_name) + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(value_name.clone()))?; + let raw: Bound<'_, PyAny> = value.getattr("value")?; + let kind: String = value.getattr("kind")?.getattr("value")?.extract()?; + let payload: Bound<'_, PyAny> = if kind == DECIMAL_KIND { + raw.str()?.into_any() + } else { + raw + }; + parameters.push(TestParameter { + name: value_name, + value_type: declaration + .getattr("value_type")? + .getattr("value")? + .extract()?, + nullable: declaration.getattr("nullable")?.is_truthy()?, + rendered: compact_json(&payload)?, + }); + } + let path: String = source_path.call_method0("as_posix")?.extract()?; + Ok(parameterized_test_label(&name, Some(&path), ¶meters)) +} + +/// The shared name column width of test rows and their expectation rows. +fn name_width(entries: &Bound<'_, PyAny>) -> PyResult { + let mut labels: Vec = Vec::new(); + let mut children: Vec = Vec::new(); + for entry in entries.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + labels.push(test_label(&entry, "name")?); + for step in entry.getattr("chain")?.try_iter()? { + let step: Bound<'_, PyAny> = step?; + if step.getattr("expected_cte_sql")?.is_truthy()? { + children.push(format!( + "expected {}", + step.getattr("model_name")?.extract::()? + )); + } + } + for assertion in entry.getattr("assertions")?.try_iter()? { + children.push(format!( + "assertion {}", + assertion?.getattr("name")?.extract::()? + )); + } + } + Ok(test_name_width(&labels, &children)) +} + +fn outcome_status(outcome: &Bound<'_, PyAny>) -> PyResult<&'static str> { + Ok(test_outcome_status( + &outcome.getattr("value")?.extract::()?, + )) +} + +/// Nested `expect` rows of a completed test, one per step. +fn expectation_rows<'py>(result: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = result.py(); + let row_class: Bound<'py, PyAny> = imported(py, PROGRESS_MODELS, "NestedProgressChildRow")?; + let name: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_name")?; + let detail: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_detail")?; + let mut rows: Vec> = Vec::new(); + for step in result.getattr("step_results")?.try_iter()? { + let step: Bound<'py, PyAny> = step?; + rows.push(call_with( + &row_class, + &[ + ("label", text(py, EXPECT_LABEL)), + ("name", name.call1((step.getattr("model_name")?,))?), + ( + "status_text", + text(py, outcome_status(&step.getattr("outcome")?)?), + ), + ("detail", detail.call1((&step,))?), + ], + )?); + } + PyTuple::new(py, rows) +} + +/// The model a planned test's progress rows are grouped under. +fn entry_group(entry: &Bound<'_, PyAny>) -> PyResult { + let mut chain: Vec<(String, bool)> = Vec::new(); + for step in entry.getattr("chain")?.try_iter()? { + let step: Bound<'_, PyAny> = step?; + chain.push(( + step.getattr("model_name")?.extract()?, + !step.getattr("expected_cte_sql")?.is_none(), + )); + } + Ok(test_group_name(&chain)) +} + +/// Forward test progress, completions and connection timing to their reporters. +#[pyclass(module = "sqlbuild._native", frozen)] +struct TestCallbackRelay { + progress: Py, + event_writer: Py, + plan_output: Py, + connection: Py, +} + +#[pymethods] +impl TestCallbackRelay { + fn on_test_start(&self, py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult<()> { + call_with( + &self.progress.bind(py).getattr("on_item_start")?, + &[ + ("group_name", text(py, &entry_group(entry)?)), + ("item_name", text(py, &test_label(entry, "name")?)), + ("canonical_resource_name", entry.getattr("name")?), + ], + )?; + Ok(()) + } + + fn on_test_complete(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let mut group: String = String::new(); + if let Some(step) = result.getattr("step_results")?.try_iter()?.next() { + group = step?.getattr("model_name")?.extract()?; + } + if group.is_empty() { + group = UNKNOWN_GROUP.to_owned(); + } + let resource_id: Bound<'_, PyAny> = call_with( + &imported(py, RESOURCE_ID_MODULE, "sql_test_resource_id")?, + &[ + ("test_name", result.getattr("test_name")?), + ("source_path", result.getattr("source_path")?), + ("block_index", result.getattr("block_index")?), + ("case_name", result.getattr("case_name")?), + ], + )?; + call_with( + &self.progress.bind(py).getattr("on_item_complete")?, + &[ + ("group_name", text(py, &group)), + ("item_name", text(py, &test_label(result, "test_name")?)), + ( + "status_text", + text(py, outcome_status(&result.getattr("outcome")?)?), + ), + ("child_rows", expectation_rows(result)?.into_any()), + ("error_code", result.getattr("error_code")?), + ("error_help", result.getattr("error_help")?), + ("error_message", result.getattr("error_message")?), + ("canonical_resource_name", result.getattr("test_name")?), + ("canonical_resource_id", resource_id), + ], + )?; + call_with( + &self.event_writer.bind(py).getattr("write_build_result")?, + &[ + ("result", result.clone()), + ("plan", self.plan_output.bind(py).clone()), + ("command", text(py, "test")), + ], + )?; + Ok(()) + } + + fn on_connection_complete( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.connection_call( + py, + "on_connection_complete", + (connection_count, elapsed_seconds), + ) + } + + fn on_connection_error( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.connection_call( + py, + "on_connection_error", + (connection_count, elapsed_seconds), + ) + } +} + +impl TestCallbackRelay { + fn connection_call( + &self, + py: Python<'_>, + method: &str, + (connection_count, elapsed_seconds): (i64, f64), + ) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("connection_count", connection_count)?; + options.set_item("elapsed_seconds", elapsed_seconds)?; + self.connection + .bind(py) + .call_method(method, (), Some(&options))?; + Ok(()) + } +} + +fn unknown_case(py: Python<'_>, message: String) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", UNKNOWN_CASE_CODE)?; + imported(py, PLANNER_ERRORS, "PlannerInputError")?.call((message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Compile the selected tests and refuse a `--case` that matched none of them. +fn compile_tests<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; + let adapter: Bound<'py, PyAny> = invocation.getattr("adapter")?; + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.getattr("effective_project_dir")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let options: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &[ + ("selected_target", request.getattr("selected_target")?), + ("no_sql_validation", request.getattr("no_sql_validation")?), + ( + "source_deferral_enabled", + PyBool::new(py, false).to_owned().into_any(), + ), + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ( + "accepts_unit_test_selectors", + PyBool::new(py, true).to_owned().into_any(), + ), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("cli_vars", request.getattr("cli_vars")?), + ("external_sql_reference_resolver", resolver), + ], + )?; + let context: Bound<'py, PyAny> = call_with( + &imported(py, STATIC_COMMAND_MODULE, "compile_static_command_context")?, + &[ + ("discovered_inputs", discovered), + ("adapter", adapter.clone()), + ("options", options), + ( + "on_progress", + invocation + .getattr("planning_progress")? + .getattr("on_progress")?, + ), + ], + )?; + let case_name: Bound<'py, PyAny> = request.getattr("case_name")?; + let plan_output: Bound<'py, PyAny> = call_with( + &imported(py, TEST_PLAN_MODULE, "build_test_command_plan")?, + &[ + ("project", context.getattr("project")?), + ("adapter", adapter), + ("scope", context.getattr("scope")?), + ("relations", context.getattr("relations")?), + ("case_name", case_name.clone()), + ], + )?; + if !case_name.is_none() && !plan_output.getattr("test_entries")?.is_truthy()? { + let cases: Vec = strings(plan_output.getattr("available_test_case_names")?)?; + let available: String = if cases.is_empty() { + "none".to_owned() + } else { + cases.join(", ") + }; + return Err(unknown_case( + py, + format!( + "SQL test case '{}' did not match the selected tests; available cases: {available}", + case_name.str()? + ), + )); + } + call_with( + &imported(py, NATIVE_MODULE, "build_static_pipeline_result")?, + &[("context", context), ("plan_output", plan_output)], + ) +} + +fn inspection_entry(entry: &Bound<'_, PyAny>) -> PyResult { + let case: Bound<'_, PyAny> = entry.getattr("case_name")?; + let name: String = entry.getattr("name")?.extract()?; + let label: String = if case.is_none() { + name + } else { + let parent: Bound<'_, PyAny> = entry.getattr("parent_name")?; + let parent: String = if parent.is_truthy()? { + parent.extract()? + } else { + name + }; + format!("{parent} [{}]", case.str()?) + }; + let mut real_models: Vec = Vec::new(); + let mut expected_models: Vec = Vec::new(); + for step in entry.getattr("chain")?.try_iter()? { + let step: Bound<'_, PyAny> = step?; + let model: String = step.getattr("model_name")?.extract()?; + if !step.getattr("expected_cte_sql")?.is_none() { + expected_models.push(model.clone()); + } + real_models.push(model); + } + Ok(TestInspectionEntry { + label, + mocked_sources: strings(entry.getattr("mock_source_names")?)?, + mocked_refs: strings(entry.getattr("mock_ref_names")?)?, + mocked_seeds: strings(entry.getattr("mock_seed_names")?)?, + mocked_dbt_refs: strings(entry.getattr("mock_dbt_ref_names")?)?, + mocked_table_functions: strings(entry.getattr("mock_table_function_names")?)?, + real_models, + expected_models, + }) +} + +/// Write the resolved plan without executing it; returns the exit code. +fn inspect(stream: &Bound<'_, PyAny>, plan_output: &Bound<'_, PyAny>) -> PyResult { + let entries: Vec = plan_output + .getattr("test_entries")? + .try_iter()? + .map(|entry| inspection_entry(&entry?)) + .collect::>()?; + let warnings: Vec = plan_output + .getattr("warnings")? + .try_iter()? + .map(|warning| { + let warning: Bound<'_, PyAny> = warning?; + Ok(TestPlanWarning { + severity: warning.getattr("severity")?.getattr("value")?.extract()?, + message: warning.getattr("message")?.str()?.extract()?, + }) + }) + .collect::>()?; + let (rendered, code) = render_test_inspection(&entries, &warnings); + stream.call_method1("write", (rendered,))?; + stream.call_method0("flush")?; + Ok(code) +} + +/// Write the ready header and run the tests with nested progress; returns the results. +fn run_tests<'py>( + invocation: &Bound<'py, PyAny>, + pipeline: &Bound<'py, PyAny>, + concurrency: i64, +) -> PyResult>> { + let py: Python<'py> = invocation.py(); + let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; + let use_color: Bound<'py, PyAny> = invocation.getattr("use_color")?; + let style: CliStyle = CliStyle { + use_color: use_color.is_truthy()?, + }; + let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let entries: Bound<'py, PyAny> = plan_output.getattr("test_entries")?; + let test_count: usize = entries.len()?; + let mut models: BTreeSet = BTreeSet::new(); + for entry in entries.try_iter()? { + for step in entry?.getattr("chain")?.try_iter()? { + models.insert(step?.getattr("model_name")?.extract()?); + } + } + let worker_count: i64 = concurrency.min(i64::try_from(test_count).unwrap_or(i64::MAX)); + let progress: Bound<'py, PyAny> = call_with( + &imported(py, NESTED_PROGRESS_MODULE, "NestedCommandProgressCallbacks")?, + &[ + ("total", PyInt::new(py, test_count).into_any()), + ("label", text(py, "test")), + ("stream", stream.clone()), + ("use_color", use_color.clone()), + ( + "name_width", + PyInt::new(py, name_width(&entries)?).into_any(), + ), + ("concurrency", PyInt::new(py, worker_count).into_any()), + ], + )?; + let projector: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + if !projector.is_none() { + for entry in entries.try_iter()? { + call_with( + &projector.getattr("expect_resource_enrichment")?, + &[("resource_name", entry?.getattr("name")?)], + )?; + } + } + stream.call_method1( + "write", + (render_test_ready((test_count, models.len()), style),), + )?; + stream.call_method0("flush")?; + let connection: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", invocation.getattr("adapter_name")?), + ("stream", stream.clone()), + ("use_color", use_color.clone()), + ], + )?; + let preflight: Bound<'py, PyAny> = call_with( + &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, + &[("stream", stream), ("use_color", use_color)], + )?; + let event_writer: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; + let relay: Bound<'py, TestCallbackRelay> = Bound::new( + py, + TestCallbackRelay { + progress: progress.unbind(), + event_writer: event_writer.clone().unbind(), + plan_output: plan_output.clone().unbind(), + connection: connection.clone().unbind(), + }, + )?; + let executed: PyResult>> = (|| { + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTOR_PIPELINE_MODELS, "TestPipelineCallbacks")?, + &[ + ( + "on_connection_start", + connection.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + relay.getattr("on_connection_complete")?, + ), + ("on_connection_error", relay.getattr("on_connection_error")?), + ( + "on_progress", + preflight.getattr("report_preflight_progress")?, + ), + ("on_test_start", relay.getattr("on_test_start")?), + ("on_test_complete", relay.getattr("on_test_complete")?), + ], + )?; + call_with( + &imported(py, EXECUTOR_RUN_MODULE, "run_test_pipeline")?, + &[ + ("plan", plan_output), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("adapter", invocation.getattr("adapter")?), + ("max_concurrency", PyInt::new(py, concurrency).into_any()), + ("callbacks", callbacks), + ("run_id", pipeline.getattr("project")?.getattr("run_id")?), + ], + )? + .try_iter()? + .collect() + })(); + let closed: PyResult> = event_writer.call_method0("close"); + let results: Vec> = executed?; + closed?; + Ok(results) +} + +/// Write the summary, failures and JSON output; returns the exit code. +fn complete( + request: &Bound<'_, PyAny>, + invocation: &Bound<'_, PyAny>, + results: &[Bound<'_, PyAny>], +) -> PyResult { + let py: Python<'_> = request.py(); + let mut failures: Vec = Vec::new(); + for result in results { + if result + .getattr("outcome")? + .getattr("value")? + .extract::()? + == PASS_OUTCOME + { + continue; + } + failures.push(TestFailure { + label: test_label(result, "test_name")?, + error_message: result.getattr("error_message")?.extract()?, + error_code: result.getattr("error_code")?.extract()?, + error_help: result.getattr("error_help")?.extract()?, + }); + } + let passed: usize = results.len() - failures.len(); + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + let style: CliStyle = CliStyle { + use_color: invocation.getattr("use_color")?.is_truthy()?, + }; + stream.call_method1( + "write", + (render_test_completion( + (passed, results.len()), + &failures, + style, + ),), + )?; + stream.call_method0("flush")?; + let tuple: Bound<'_, PyAny> = PyTuple::new(py, results)?.into_any(); + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_test_execution_json")?, + &[("results", tuple)], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(test_exit_code(passed, results.len())) +} + +/// Run one `sqb test` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let inspect_only: bool = request.getattr("inspect")?.is_truthy()?; + let invocation: Bound<'_, PyAny> = call_with( + &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, + &[ + ("request", request.clone()), + ( + "invocation_type", + imported(py, COMMAND_MODELS, "TestInvocation")?, + ), + ( + "require_connection", + PyBool::new(py, !inspect_only).to_owned().into_any(), + ), + ], + )?; + let pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + if inspect_only { + return inspect(&stream, &pipeline.getattr("plan_output")?); + } + let requested: Bound<'_, PyAny> = request.getattr("concurrency")?; + let concurrency: i64 = if requested.is_none() { + pipeline + .getattr("project")? + .getattr("settings")? + .getattr("concurrency")? + .extract()? + } else { + requested.extract()? + }; + stream.call_method1("write", ("\n",))?; + let header: Bound<'_, PyDict> = PyDict::new(py); + header.set_item("stream", &stream)?; + header.set_item("command", EXECUTION_COMMAND)?; + header.set_item("target", py.None())?; + header.set_item("concurrency", concurrency)?; + header.set_item("use_color", invocation.getattr("use_color")?)?; + imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&header))?; + let results: Vec> = run_tests(&invocation, &pipeline, concurrency)?; + call_with( + &imported(py, RUNTIME_TARGET_MODULE, "write_test_runtime_target")?, + &[ + ( + "target_dir", + invocation + .getattr("effective_project_dir")? + .call_method1("__truediv__", ("target",))?, + ), + ("adapter", invocation.getattr("adapter")?), + ("plan_output", pipeline.getattr("plan_output")?), + ("results", PyTuple::new(py, &results)?.into_any()), + ], + )?; + complete(request, &invocation, &results) +} + +/// Add source and safe typed values only for parameterized test rows. +#[pyfunction] +#[pyo3(signature = (**options))] +fn format_parameterized_test_label( + py: Python<'_>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "format_parameterized_test_label", + options, + ( + &[ + "name", + "source_path", + "parameter_schema", + "parameter_values", + ], + &[], + ), + )?; + let item: Bound<'_, PyAny> = py.import("types")?.getattr("SimpleNamespace")?.call( + (), + Some(&{ + let fields: Bound<'_, PyDict> = PyDict::new(py); + for field in [ + "name", + "source_path", + "parameter_schema", + "parameter_values", + ] { + fields.set_item(field, arguments.value(field)?)?; + } + fields + }), + )?; + test_label(&item, "name") +} + +/// Resolve one shared width for test rows and nested expectation rows. +#[pyfunction] +fn resolve_test_name_width(test_entries: &Bound<'_, PyAny>) -> PyResult { + name_width(test_entries) +} + +/// Build nested expectation rows for a completed SQL unit test. +#[pyfunction] +fn build_test_expectation_rows<'py>(result: &Bound<'py, PyAny>) -> PyResult> { + expectation_rows(result) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_test_command, module)?)?; + module.add_function(wrap_pyfunction!(format_parameterized_test_label, module)?)?; + module.add_function(wrap_pyfunction!(resolve_test_name_width, module)?)?; + module.add_function(wrap_pyfunction!(build_test_expectation_rows, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs new file mode 100644 index 0000000000..5b0c5d4001 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs @@ -0,0 +1,164 @@ +//! Migration event IDs and the SQL that appends and looks up one event row. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_state::events::errors::StateSqlError; +use sqlbuild_state::events::main::column_migration_event_id::column_migration_event_id; +use sqlbuild_state::events::main::event_exists_sql::event_exists_sql; +use sqlbuild_state::events::main::event_insert_sql::event_insert_sql; +use sqlbuild_state::events::main::migration_event_id::migration_event_id; +use sqlbuild_state::events::main::old_name_view_event_id::old_name_view_event_id; +use sqlbuild_state::events::models::{EventRelation, StateCell, StateColumnType, StateValue}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, presumed_utc}; + +const SQL_VALUES_EXCEPTIONS: &str = "sqlbuild.sql_values.exceptions"; +const STRING_TYPE: &str = "string"; +const TEXT_TIMESTAMP_TYPE: &str = "text_timestamp"; + +fn emit_error(error: JsonEmitError) -> PyErr { + PyValueError::new_err(format!("{error:?}")) +} + +fn relation(value: &Bound<'_, PyAny>) -> PyResult { + Ok(EventRelation { + database: value.getattr("database")?.extract()?, + schema: value.getattr("schema")?.extract()?, + name: value.getattr("name")?.extract()?, + }) +} + +/// The content-addressed ID for one migration performed by one run. +#[pyfunction(name = "deterministic_migration_event_id")] +#[pyo3(signature = (*, run_id, target_name, origin, destination))] +fn migration_id( + run_id: &str, + target_name: Option<&str>, + origin: &Bound<'_, PyAny>, + destination: &Bound<'_, PyAny>, +) -> PyResult { + migration_event_id( + (run_id, target_name), + &relation(origin)?, + &relation(destination)?, + ) + .map_err(emit_error) +} + +/// The content-addressed ID for one column rename performed by one run. +#[pyfunction(name = "deterministic_column_migration_event_id")] +#[pyo3(signature = (**request))] +fn column_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("deterministic_column_migration_event_id", request)?; + let run_id: String = request.get("run_id")?.extract()?; + let target_name: Option = request.get("target_name")?.extract()?; + let origin_column: String = request.get("origin_column")?.extract()?; + let destination_column: String = request.get("destination_column")?.extract()?; + column_migration_event_id( + (&run_id, target_name.as_deref()), + &relation(&request.get("relation")?)?, + (&origin_column, &destination_column), + ) + .map_err(emit_error) +} + +/// The content-addressed ID of one old-name step of one recorded move. +#[pyfunction(name = "deterministic_old_name_view_event_id")] +#[pyo3(signature = (*, event_type, migration_event_id))] +fn old_name_id(event_type: &Bound<'_, PyAny>, migration_event_id: &str) -> PyResult { + let event_type: String = event_type.getattr("value")?.extract()?; + old_name_view_event_id(&event_type, migration_event_id).map_err(emit_error) +} + +/// The lookup that makes an append-only event write idempotent. +#[pyfunction(name = "render_event_exists_sql")] +#[pyo3(signature = (*, table, event_id))] +fn exists_sql(table: &str, event_id: &str) -> String { + event_exists_sql(table, event_id) +} + +fn state_value(value: &Bound<'_, PyAny>, column_type: StateColumnType) -> PyResult { + if value.is_none() { + return Ok(StateValue::Null); + } + if let Ok(text) = value.downcast::() { + return Ok(StateValue::Text(text.extract()?)); + } + if column_type == StateColumnType::TextTimestamp && date_time_typed(value)? { + return Ok(StateValue::UtcTimestamp(presumed_utc(value)?)); + } + Ok(StateValue::Other) +} + +fn column_type(declared: &Bound<'_, PyAny>) -> PyResult { + let name: String = declared.str()?.extract()?; + match name.as_str() { + STRING_TYPE => Ok(StateColumnType::String), + TEXT_TIMESTAMP_TYPE => Ok(StateColumnType::TextTimestamp), + _ => Err(PyValueError::new_err(format!( + "migration event columns are string or text timestamps, not {name}" + ))), + } +} + +/// One append-only event row as `INSERT ... VALUES` with typed literals. +#[pyfunction(name = "render_event_insert_sql")] +#[pyo3(signature = (*, table, columns, column_types, values))] +fn insert_sql( + table: &str, + columns: Vec, + column_types: &Bound<'_, PyAny>, + values: Vec>, +) -> PyResult { + if columns.len() != values.len() { + return Err(PyValueError::new_err( + "event columns and values must have the same length", + )); + } + let mut cells: Vec = Vec::with_capacity(columns.len()); + for (column, value) in columns.into_iter().zip(&values) { + let column_type: StateColumnType = column_type(&column_types.get_item(&column)?)?; + let converted: PyResult = state_value(value, column_type); + let value: StateValue = match converted { + Ok(value) => value, + Err(error) => { + event_insert_sql(table, &cells).map_err(rendering_error(value.py()))?; + return Err(error); + } + }; + cells.push(StateCell { + column, + column_type, + value, + }); + } + event_insert_sql(table, &cells).map_err(rendering_error(column_types.py())) +} + +/// `SqlValueRenderingError(message)` for an ill-typed value. +fn rendering_error(py: Python<'_>) -> impl Fn(StateSqlError) -> PyErr + '_ { + move |error: StateSqlError| { + let raised = imported(py, SQL_VALUES_EXCEPTIONS, "SqlValueRenderingError") + .and_then(|class| class.call1((error.0,))); + match raised { + Ok(exception) => PyErr::from_value(exception), + Err(failure) => failure, + } + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(migration_id, module)?)?; + module.add_function(wrap_pyfunction!(column_id, module)?)?; + module.add_function(wrap_pyfunction!(old_name_id, module)?)?; + module.add_function(wrap_pyfunction!(exists_sql, module)?)?; + module.add_function(wrap_pyfunction!(insert_sql, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs new file mode 100644 index 0000000000..a2377ff690 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs @@ -0,0 +1,237 @@ +//! Source freshness data versions, their hashes, skip equivalence and age policy. + +use std::cell::RefCell; + +use pyo3::exceptions::PyOverflowError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyInt, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_state::freshness::errors::FreshnessError; +use sqlbuild_state::freshness::main::age_status::age_status; +use sqlbuild_state::freshness::main::data_version_hash::data_version_hash; +use sqlbuild_state::freshness::main::normalized_data_version::normalized_data_version; +use sqlbuild_state::freshness::main::records_equivalent::records_equivalent; +use sqlbuild_state::freshness::models::{AgeStatus, ComparableRecord, DataVersion}; +use sqlbuild_state::freshness::types::TimestampParser; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::values::{DATETIME_MODULE, local_date_time}; +use crate::bindings::_helpers::state_rows::timestamps::{ + date_time_typed, in_utc, offset_aware, presumed_utc, +}; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.source_freshness.exceptions"; +const TYPES_MODULE: &str = "sqlbuild.compiler.source_freshness.types"; +const TIMESTAMP_KIND: &str = "timestamp"; +const INTEGER_KIND: &str = "integer"; +const STRING_KIND: &str = "string"; + +fn raised(py: Python<'_>, class: &str, message: String) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, class).and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn python_error(py: Python<'_>, error: FreshnessError) -> PyErr { + match error { + FreshnessError::Observation(message) => { + raised(py, "SourceFreshnessObservationError", message) + } + FreshnessError::Input(message) => raised(py, "SourceFreshnessInputError", message), + FreshnessError::Overflow(message) => PyOverflowError::new_err(message), + } +} + +/// Hash the stable source freshness identity used as a graph input. +#[pyfunction(name = "source_freshness_data_version_hash")] +#[pyo3(signature = (*, source_name, strategy, value_kind, data_version))] +fn version_hash( + source_name: &str, + strategy: &str, + value_kind: &str, + data_version: &str, +) -> PyResult { + data_version_hash((source_name, strategy), value_kind, data_version) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(format!("{error:?}"))) +} + +fn observation_error(py: Python<'_>, message: &str) -> PyErr { + raised(py, "SourceFreshnessObservationError", message.to_owned()) +} + +/// A source freshness value normalized for stable state storage. +#[pyfunction(name = "normalize_source_freshness_data_version")] +#[pyo3(signature = (*, value, value_kind))] +fn normalize<'py>( + value: &Bound<'py, PyAny>, + value_kind: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = value.py(); + if value_kind.eq(TIMESTAMP_KIND)? { + if !date_time_typed(value)? { + return Err(observation_error( + py, + "Timestamp source freshness values must be datetime values", + )); + } + let aware: bool = !value.getattr("tzinfo")?.is_none(); + let local: LocalDateTime = if aware { + local_date_time(&in_utc(value)?)? + } else { + local_date_time(value)? + }; + let text: String = normalized_data_version(&DataVersion::Timestamp { local, utc: aware }); + return Ok(PyString::new(py, &text).into_any()); + } + if value_kind.eq(INTEGER_KIND)? { + if value.is_instance_of::() || !value.is_instance_of::() { + return Err(observation_error( + py, + "Integer source freshness values must be integer values", + )); + } + return Ok(value.str()?.into_any()); + } + if value_kind.eq(STRING_KIND)? { + if !value.is_instance_of::() { + return Err(observation_error( + py, + "String source freshness values must be string values", + )); + } + return Ok(value.clone()); + } + Err(raised( + py, + "SourceFreshnessObservationError", + format!( + "Unsupported source freshness value kind: {}", + value_kind.str()? + ), + )) +} + +/// Stored timestamp text parsed with `datetime.fromisoformat`, remembering a Python failure. +struct StateTimestamps<'py> { + py: Python<'py>, + failure: RefCell>, +} + +impl StateTimestamps<'_> { + fn parsed(&self, text: &str) -> PyResult { + let parsed = + imported(self.py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (text,)); + match parsed { + Ok(value) => presumed_utc(&value), + Err(error) if error.is_instance_of::(self.py) => { + let wrapped: PyErr = observation_error( + self.py, + &format!( + "Timestamp source freshness state value is not valid ISO datetime: {text}" + ), + ); + wrapped.set_cause(self.py, Some(error)); + Err(wrapped) + } + Err(error) => Err(error), + } + } +} + +impl TimestampParser for StateTimestamps<'_> { + fn utc_timestamp(&self, text: &str) -> Result { + self.parsed(text).map_err(|error| { + let message: String = error.to_string(); + *self.failure.borrow_mut() = Some(error); + FreshnessError::Observation(message) + }) + } +} + +fn comparable(record: &Bound<'_, PyAny>) -> PyResult { + Ok(ComparableRecord { + data_version_hash: record.getattr("data_version_hash")?.extract()?, + value_kind: record.getattr("value_kind")?.extract()?, + data_version: record.getattr("data_version")?.extract()?, + }) +} + +/// Whether two source freshness records are equivalent for skip decisions. +#[pyfunction(name = "source_freshness_records_equivalent")] +#[pyo3(signature = (*, previous_record, current_record, lag_tolerance = None))] +fn equivalent( + previous_record: &Bound<'_, PyAny>, + current_record: &Bound<'_, PyAny>, + lag_tolerance: Option<&str>, +) -> PyResult { + let py: Python<'_> = previous_record.py(); + let parser = StateTimestamps { + py, + failure: RefCell::new(None), + }; + let records = (&comparable(previous_record)?, &comparable(current_record)?); + records_equivalent(records, lag_tolerance, &parser).map_err(|error| { + parser + .failure + .borrow_mut() + .take() + .unwrap_or_else(|| python_error(py, error)) + }) +} + +/// A `datetime` required to be aware, in UTC. +fn aware_utc(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult { + if !offset_aware(value)? { + return Err(observation_error( + value.py(), + &format!("source freshness {field_name} must be timezone-aware"), + )); + } + local_date_time(&in_utc(value)?) +} + +/// Optional timestamp age policy for one source freshness observation. +#[pyfunction(name = "evaluate_source_freshness_age_policy")] +#[pyo3(signature = (*, policy, data_version, observed_at))] +fn age_policy<'py>( + policy: &Bound<'py, PyAny>, + data_version: &Bound<'py, PyAny>, + observed_at: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = policy.py(); + if policy.is_none() { + return Ok(None); + } + let statuses: Bound<'py, PyAny> = imported(py, TYPES_MODULE, "SourceFreshnessAgeStatus")?; + if !date_time_typed(data_version)? { + return statuses.getattr("UNKNOWN").map(Some); + } + let version: LocalDateTime = aware_utc(data_version, "data_version")?; + let observed: LocalDateTime = aware_utc(observed_at, "observed_at")?; + let error_after: Option = policy.getattr("error_after")?.extract()?; + let warn_after: Option = policy.getattr("warn_after")?.extract()?; + let status: AgeStatus = age_status( + (&version, &observed), + error_after.as_deref(), + warn_after.as_deref(), + ) + .map_err(|error| python_error(py, error))?; + let member: &str = match status { + AgeStatus::Pass => "PASS", + AgeStatus::Warn => "WARN", + AgeStatus::Error => "ERROR", + }; + statuses.getattr(member).map(Some) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(version_hash, module)?)?; + module.add_function(wrap_pyfunction!(normalize, module)?)?; + module.add_function(wrap_pyfunction!(equivalent, module)?)?; + module.add_function(wrap_pyfunction!(age_policy, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs new file mode 100644 index 0000000000..936e0ba0fa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod events; +pub(crate) mod freshness; +pub(crate) mod registration; +pub(crate) mod timestamps; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs new file mode 100644 index 0000000000..c06548943e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs @@ -0,0 +1,11 @@ +//! Register the native warehouse state row functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::state_rows::{events, freshness}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + events::register(module)?; + freshness::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs new file mode 100644 index 0000000000..b679778b32 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/timestamps.rs @@ -0,0 +1,33 @@ +//! Python `datetime` values normalized to UTC wall time. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::values::{DATETIME_MODULE, local_date_time}; + +/// Whether `value` is a `datetime`. +pub(crate) fn date_time_typed(value: &Bound<'_, PyAny>) -> PyResult { + value.is_instance(&imported(value.py(), DATETIME_MODULE, "datetime")?) +} + +/// Whether `value` has a `tzinfo` that reports an offset. +pub(crate) fn offset_aware(value: &Bound<'_, PyAny>) -> PyResult { + Ok(!value.getattr("tzinfo")?.is_none() && !value.call_method0("utcoffset")?.is_none()) +} + +/// `value.astimezone(UTC)`. +pub(crate) fn in_utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + value.call_method1( + "astimezone", + (imported(value.py(), DATETIME_MODULE, "UTC")?,), + ) +} + +/// The UTC wall time of `value`, reading a naive or offset-less value as UTC. +pub(crate) fn presumed_utc(value: &Bound<'_, PyAny>) -> PyResult { + if offset_aware(value)? { + return local_date_time(&in_utc(value)?); + } + local_date_time(value) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 70d923473f..3037c000d6 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,12 +5,11 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, - command_runtime, compile_attachment, compile_command, compile_driver, compile_outputs, - compiled_project, contracts, cost_report, cursor, dag_artifact, debug_command, discovery, - fingerprints, freshness, frontier, graph, janitor, lineage, macro_bridge, manifest, - model_config, plan_output, presentation_text, project_assembly, refactor, references, render, - result_output, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, - type_system, workspace_commands, + compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, + contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, + lineage, macro_bridge, manifest, model_config, plan_output, project_assembly, refactor, + references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, + state_rows, type_system, }; use crate::bindings::models; @@ -45,19 +44,13 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { cache::registration::register(module)?; attachments::registration::register(module)?; refactor::registration::register(module)?; - janitor::registration::register(module)?; - freshness::registration::register(module)?; plan_output::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; - result_output::registration::register(module)?; - cost_report::registration::register(module)?; - debug_command::registration::register(module)?; - command_runtime::registration::register(module)?; - presentation_text::registration::register(module)?; - workspace_commands::registration::register(module)?; auditing::registration::register(module)?; cursor::registration::register(module)?; + state_rows::registration::register(module)?; + diagnostics::redaction::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index f36c31c00f..a7d43d7096 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -2,11 +2,6 @@ use crate::models::RulesConfig; use fensu_policy::policy::types::RuleCodeGrammar; use std::path::Path; -pub(crate) fn load_config_json(project_dir: &Path) -> Result { - let config = load(project_dir)?; - serde_json::to_string(&config).map_err(|error| error.to_string()) -} - pub(crate) fn load(project_dir: &Path) -> Result { let path = project_dir.join("sqlbuild_project.toml"); let source = std::fs::read_to_string(&path).map_err(|error| { diff --git a/crates/sqlbuild-rules/src/configuration/errors.rs b/crates/sqlbuild-rules/src/configuration/errors.rs new file mode 100644 index 0000000000..cd92448b71 --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/errors.rs @@ -0,0 +1,8 @@ +//! Rules configuration that a project's own policy rejects. + +/// A configuration policy violation, with help showing the setting to change. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConfigPolicyError { + pub message: String, + pub help: String, +} diff --git a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs new file mode 100644 index 0000000000..7108ee1055 --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs @@ -0,0 +1,57 @@ +//! `forbidden_exceptions`: exceptions or scoped ignores in a project that disallows them. + +use sqlbuild_analysis::contracts::main::join_helps::join_helps; +use sqlbuild_core::cli_text::main::format_count_noun::format_count_noun; + +use crate::configuration::errors::ConfigPolicyError; +use crate::configuration::main::setting_help::setting_help; +use crate::constants::PROJECT_CONFIG_FILE; +use crate::models::RulesConfig; + +/// The policy error when `allow_exceptions = false` and exception or ignore entries exist. +#[must_use] +pub fn forbidden_exceptions(config: &RulesConfig) -> Option { + if config.allow_exceptions { + return None; + } + let mut entries: Vec = Vec::new(); + let mut tables: Vec = Vec::new(); + for (key, count) in [ + ("rule_exceptions", config.rule_exceptions.len()), + ("rule_ignores", config.rule_ignores.len()), + ] { + if count == 0 { + continue; + } + let table: String = format!("[[rules.{key}]]"); + let count: i64 = i64::try_from(count).unwrap_or(i64::MAX); + entries.push(format_count_noun( + count, + &format!("{table} entry"), + Some(&format!("{table} entries")), + )); + tables.push(table); + } + if entries.is_empty() { + return None; + } + let message: String = format!( + "{PROJECT_CONFIG_FILE} has {}, which this project does not allow; \ + {PROJECT_CONFIG_FILE} sets [rules] allow_exceptions = false", + entries.join(" and ") + ); + let help: String = join_helps(&[ + format!( + "remove every {} entry and fix the findings it hides; to turn a Rule off for the \ + whole project, add its code to [rules] ignore", + tables.join(" and ") + ), + setting_help( + "to allow Rule exceptions, scoped ignores and inline suppressions", + "rules", + "allow_exceptions", + "true", + ), + ]); + Some(ConfigPolicyError { message, help }) +} diff --git a/crates/sqlbuild-rules/src/configuration/main/load.rs b/crates/sqlbuild-rules/src/configuration/main/load.rs deleted file mode 100644 index 43ca577d63..0000000000 --- a/crates/sqlbuild-rules/src/configuration/main/load.rs +++ /dev/null @@ -1,5 +0,0 @@ -use std::path::Path; - -pub fn load_config_json(project_dir: &Path) -> Result { - crate::configuration::_helpers::loading::load_config_json(project_dir) -} diff --git a/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs b/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs new file mode 100644 index 0000000000..2c3ba017d0 --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs @@ -0,0 +1,10 @@ +//! `load_rules_config`: the validated `[rules]` configuration of one project. + +use std::path::Path; + +use crate::models::RulesConfig; + +/// The project's rules configuration, or the reason it cannot be loaded. +pub fn load_rules_config(project_dir: &Path) -> Result { + crate::configuration::_helpers::loading::load(project_dir) +} diff --git a/crates/sqlbuild-rules/src/configuration/main/mod.rs b/crates/sqlbuild-rules/src/configuration/main/mod.rs index fbd112cce3..42488f2f4f 100644 --- a/crates/sqlbuild-rules/src/configuration/main/mod.rs +++ b/crates/sqlbuild-rules/src/configuration/main/mod.rs @@ -1,4 +1,5 @@ -pub mod load; +pub mod forbidden_exceptions; +pub mod load_rules_config; pub mod setting_help; pub mod toml_string_array; pub mod validate; diff --git a/crates/sqlbuild-rules/src/configuration/mod.rs b/crates/sqlbuild-rules/src/configuration/mod.rs index 0b794b2977..3e943c42e4 100644 --- a/crates/sqlbuild-rules/src/configuration/mod.rs +++ b/crates/sqlbuild-rules/src/configuration/mod.rs @@ -1,2 +1,3 @@ pub mod _helpers; +pub mod errors; pub mod main; diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index bbb7d6c2f7..f2b6b08109 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -32,3 +32,4 @@ pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; pub(crate) const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v1"; +pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/current.rs b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/current.rs new file mode 100644 index 0000000000..19ab2197e4 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/current.rs @@ -0,0 +1,25 @@ +//! The stored results that still belong to the current rule and subject identities. + +use crate::custom_rule_cache::models::{CustomRule, StoredEntry, StoredRule}; +use crate::custom_rule_cache::types::StoredRules; + +/// The stored rule cache of `rule` when its identity is unchanged. +pub(crate) fn current_rule<'a>( + stored: &'a StoredRules, + rule: &CustomRule, +) -> Option<&'a StoredRule> { + let cached: &StoredRule = stored.get(&rule.code)?; + (rule.identity.as_deref() == Some(cached.identity.as_str())).then_some(cached) +} + +/// The stored entry for one subject when its identity is unchanged. +pub(crate) fn current_entry<'a>( + cached: &'a StoredRule, + key: &str, + identity: &str, +) -> Option<&'a StoredEntry> { + cached + .entries + .get(key) + .filter(|entry| entry.subject_identity == identity) +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs new file mode 100644 index 0000000000..720e15600a --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs @@ -0,0 +1,27 @@ +//! SHA-256 over length-prefixed fields, so no two field lists share an identity. + +use sha2::{Digest, Sha256}; + +use crate::constants::NATIVE_BUILD_IDENTITY; +use crate::custom_rule_cache::constants::CACHE_VERSION; + +/// The hex digest of the cache version, the native build and each field in order. +pub(crate) fn field_digest(kind: &str, fields: &[&str]) -> String { + let mut digest = Sha256::new(); + for field in [CACHE_VERSION, NATIVE_BUILD_IDENTITY, kind] + .iter() + .chain(fields.iter()) + { + digest.update((field.len() as u64).to_le_bytes()); + digest.update(field.as_bytes()); + } + format!("{:x}", digest.finalize()) +} + +/// An optional field as a presence flag and its text. +pub(crate) fn optional_fields(value: Option<&str>) -> [&str; 2] { + match value { + Some(text) => ["1", text], + None => ["0", ""], + } +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/mod.rs b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/mod.rs new file mode 100644 index 0000000000..4bd43b6a42 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod current; +pub(crate) mod digest; +pub(crate) mod stored; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/stored.rs b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/stored.rs new file mode 100644 index 0000000000..1f2457e97e --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/stored.rs @@ -0,0 +1,53 @@ +//! Stored rule caches decoded from their JSON form. + +use std::collections::BTreeMap; + +use serde_json::{Map, Value}; + +use crate::custom_rule_cache::constants::{ENTRIES_KEY, IDENTITY_KEY, READSETS_KEY}; +use crate::custom_rule_cache::models::{StoredEntry, StoredRule}; +use crate::custom_rule_cache::types::{FactKey, ReadSet}; + +fn strings(value: &Value) -> Option { + value + .as_array()? + .iter() + .map(|part| part.as_str().map(str::to_owned)) + .collect() +} + +fn readset(value: &Value) -> Option { + value.as_array()?.iter().map(strings).collect() +} + +fn entry(value: &Value) -> Option { + let [subject_identity, readset, reads_digest, findings] = value.as_array()?.as_slice() else { + return None; + }; + Some(StoredEntry { + subject_identity: subject_identity.as_str()?.to_owned(), + readset: readset.as_u64()? as usize, + reads_digest: reads_digest.as_str()?.to_owned(), + findings: findings.as_array()?.clone(), + }) +} + +pub(crate) fn stored_rule(value: &Value) -> Option { + let object: &Map = value.as_object()?; + let entries: BTreeMap = object + .get(ENTRIES_KEY)? + .as_object()? + .iter() + .map(|(subject, value)| Some((subject.clone(), entry(value)?))) + .collect::>()?; + Some(StoredRule { + identity: object.get(IDENTITY_KEY)?.as_str()?.to_owned(), + readsets: object + .get(READSETS_KEY)? + .as_array()? + .iter() + .map(readset) + .collect::>()?, + entries, + }) +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs b/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs new file mode 100644 index 0000000000..18d9b415af --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs @@ -0,0 +1,18 @@ +//! The custom-rule result cache format and the inputs its identities separate. + +/// Bumped whenever identity derivation or the stored entry shape changes. +pub(crate) const CACHE_VERSION: &str = "custom-rules-native-v1"; +pub(crate) const RULE_KIND: &str = "rule"; +pub(crate) const PROJECT_SUBJECT_KIND: &str = "project"; +pub(crate) const MODEL_SUBJECT_KIND: &str = "model"; +/// The subject key of a project-wide rule's single invocation. +pub(crate) const PROJECT_SUBJECT: &str = ""; +pub(crate) const PROJECT_SUBJECT_NAME: &str = "project"; +pub(crate) const MODEL_SUBJECT_NAME: &str = "model"; +pub(crate) const VERSION_KEY: &str = "version"; +pub(crate) const RULES_KEY: &str = "rules"; +pub(crate) const IDENTITY_KEY: &str = "identity"; +pub(crate) const READSETS_KEY: &str = "readsets"; +pub(crate) const ENTRIES_KEY: &str = "entries"; +pub(crate) const CODE_KEY: &str = "code"; +pub(crate) const PATH_KEY: &str = "path"; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/errors.rs b/crates/sqlbuild-rules/src/custom_rule_cache/errors.rs new file mode 100644 index 0000000000..0837df974a --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/errors.rs @@ -0,0 +1,5 @@ +//! Custom-rule results that contradict the rules that produced them. + +/// A host returned a finding outside the rule or subject it evaluated. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CustomFindingError(pub String); diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/custom_rule_cache_text.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/custom_rule_cache_text.rs new file mode 100644 index 0000000000..b41fac8c4d --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/custom_rule_cache_text.rs @@ -0,0 +1,60 @@ +//! `custom_rule_cache_text`: the stored form of the custom-rule result cache. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Map, Value, json}; + +use crate::custom_rule_cache::constants::{ + CACHE_VERSION, ENTRIES_KEY, IDENTITY_KEY, READSETS_KEY, RULES_KEY, VERSION_KEY, +}; +use crate::custom_rule_cache::models::{CustomRule, StoredRule}; +use crate::custom_rule_cache::types::StoredRules; + +fn rule_value(cached: &StoredRule, rule: &CustomRule) -> Value { + let live: BTreeSet<&str> = rule + .subjects + .iter() + .map(|subject| subject.key.as_str()) + .collect(); + let used: BTreeSet = cached + .entries + .iter() + .filter(|(key, _)| live.contains(key.as_str())) + .map(|(_, entry)| entry.readset) + .collect(); + let remap: BTreeMap = used + .iter() + .enumerate() + .map(|(new, old)| (*old, new)) + .collect(); + let entries: Map = cached + .entries + .iter() + .filter_map(|(key, entry)| { + let index: usize = *remap.get(&entry.readset)?; + let value = json!([ + entry.subject_identity, + index, + entry.reads_digest, + entry.findings + ]); + live.contains(key.as_str()).then(|| (key.clone(), value)) + }) + .collect(); + let readsets: Vec = used + .iter() + .filter_map(|index| cached.readsets.get(*index)) + .map(|readset| json!(readset)) + .collect(); + json!({ IDENTITY_KEY: cached.identity, READSETS_KEY: readsets, ENTRIES_KEY: entries }) +} + +/// Compact JSON of the format version and each selected rule's live subjects and read sets. +#[must_use] +pub fn custom_rule_cache_text(stored: &StoredRules, rules: &[CustomRule]) -> String { + let stored_rules: Map = rules + .iter() + .filter_map(|rule| Some((rule.code.clone(), rule_value(stored.get(&rule.code)?, rule)))) + .collect(); + json!({ VERSION_KEY: CACHE_VERSION, RULES_KEY: stored_rules }).to_string() +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs new file mode 100644 index 0000000000..7e05b58abf --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs @@ -0,0 +1,54 @@ +//! `lookup_custom_rule_cache`: reuse results whose recorded fact reads still digest the same. + +use std::collections::BTreeMap; + +use crate::custom_rule_cache::_helpers::current::{current_entry, current_rule}; +use crate::custom_rule_cache::models::{CustomCacheLookup, CustomRule, StoredEntry, StoredRule}; +use crate::custom_rule_cache::types::{ReadSet, StoredRules}; + +fn reusable<'a>( + cached: Option<&'a StoredRule>, + rule_subject: (&str, &str), + digests: &BTreeMap, +) -> Option<&'a StoredEntry> { + let cached: &StoredRule = cached?; + let entry: &StoredEntry = current_entry(cached, rule_subject.0, rule_subject.1)?; + let readset: &ReadSet = cached.readsets.get(entry.readset)?; + (digests.get(readset) == Some(&entry.reads_digest)).then_some(entry) +} + +/// Hits per rule and subject, and each rule's missing subjects (`None` when all miss). +#[must_use] +pub fn lookup_custom_rule_cache( + stored: &StoredRules, + rules: &[CustomRule], + digests: &BTreeMap, +) -> CustomCacheLookup { + let mut lookup = CustomCacheLookup::default(); + for rule in rules { + let cached: Option<&StoredRule> = current_rule(stored, rule); + let mut missing: Vec = Vec::new(); + for subject in &rule.subjects { + match reusable(cached, (&subject.key, &subject.identity), digests) { + Some(entry) => { + let key = (rule.code.clone(), subject.key.clone()); + lookup.hits.push((key, entry.findings.clone())); + lookup.hit_count += 1; + } + None => missing.push(subject.key.clone()), + } + } + if missing.is_empty() { + continue; + } + lookup.miss_count += missing.len(); + let planned: Option> = if missing.len() == rule.subjects.len() { + None + } else { + missing.sort(); + Some(missing) + }; + lookup.plan.insert(rule.code.clone(), planned); + } + lookup +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/mod.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/mod.rs new file mode 100644 index 0000000000..21874c954a --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/mod.rs @@ -0,0 +1,9 @@ +pub mod custom_rule_cache_text; +pub mod lookup_custom_rule_cache; +pub mod ordered_custom_findings; +pub mod parse_custom_rule_cache; +pub mod pending_readsets; +pub mod rule_identity; +pub mod rule_subjects; +pub mod store_custom_results; +pub mod validate_custom_findings; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/ordered_custom_findings.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/ordered_custom_findings.rs new file mode 100644 index 0000000000..4aee088c65 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/ordered_custom_findings.rs @@ -0,0 +1,20 @@ +//! `ordered_custom_findings`: custom findings in rule code, then subject order. + +use serde_json::Value; + +use crate::custom_rule_cache::models::CustomRule; +use crate::custom_rule_cache::types::SubjectResults; + +/// Every result's findings, rule by rule and subject by subject. +#[must_use] +pub fn ordered_custom_findings(rules: &[CustomRule], results: &SubjectResults) -> Vec { + let mut findings: Vec = Vec::new(); + for rule in rules { + for subject in &rule.subjects { + if let Some(values) = results.get(&(rule.code.clone(), subject.key.clone())) { + findings.extend(values.iter().cloned()); + } + } + } + findings +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/parse_custom_rule_cache.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/parse_custom_rule_cache.rs new file mode 100644 index 0000000000..61bc93d702 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/parse_custom_rule_cache.rs @@ -0,0 +1,25 @@ +//! `parse_custom_rule_cache`: the usable rule caches of a stored custom-rule cache. + +use serde_json::Value; + +use crate::custom_rule_cache::_helpers::stored::stored_rule; +use crate::custom_rule_cache::constants::{CACHE_VERSION, RULES_KEY, VERSION_KEY}; +use crate::custom_rule_cache::types::StoredRules; + +/// Rule caches written by this format; malformed rules are skipped and anything else is empty. +#[must_use] +pub fn parse_custom_rule_cache(text: &str) -> StoredRules { + let Ok(Value::Object(payload)) = serde_json::from_str::(text) else { + return StoredRules::new(); + }; + if payload.get(VERSION_KEY).and_then(Value::as_str) != Some(CACHE_VERSION) { + return StoredRules::new(); + } + let Some(Value::Object(rules)) = payload.get(RULES_KEY) else { + return StoredRules::new(); + }; + rules + .iter() + .filter_map(|(code, value)| Some((code.clone(), stored_rule(value)?))) + .collect() +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/pending_readsets.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/pending_readsets.rs new file mode 100644 index 0000000000..2b5af914b6 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/pending_readsets.rs @@ -0,0 +1,24 @@ +//! `pending_readsets`: the stored read sets whose current digest decides a cache hit. + +use crate::custom_rule_cache::_helpers::current::{current_entry, current_rule}; +use crate::custom_rule_cache::models::CustomRule; +use crate::custom_rule_cache::types::{ReadSet, StoredRules}; + +/// Distinct read sets of every reusable entry, in first-use order. +#[must_use] +pub fn pending_readsets(stored: &StoredRules, rules: &[CustomRule]) -> Vec { + let mut pending: Vec = Vec::new(); + for rule in rules { + let Some(cached) = current_rule(stored, rule) else { + continue; + }; + for subject in &rule.subjects { + let readset: Option<&ReadSet> = current_entry(cached, &subject.key, &subject.identity) + .and_then(|entry| cached.readsets.get(entry.readset)); + if let Some(readset) = readset.filter(|readset| !pending.contains(readset)) { + pending.push(readset.clone()); + } + } + } + pending +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_identity.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_identity.rs new file mode 100644 index 0000000000..987b8f9a80 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_identity.rs @@ -0,0 +1,29 @@ +//! `rule_identity`: everything besides fact reads that keys one custom rule's stored results. + +use crate::custom_rule_cache::_helpers::digest::{field_digest, optional_fields}; +use crate::custom_rule_cache::constants::RULE_KIND; +use crate::custom_rule_cache::models::RuleIdentityInput; + +/// A digest of the code identity, host runtime, rule metadata and source, config, dialect and root. +#[must_use] +pub fn rule_identity(input: &RuleIdentityInput) -> String { + let [has_subject, subject] = optional_fields(input.subject.as_deref()); + field_digest( + RULE_KIND, + &[ + &input.code_identity, + &input.runtime_version, + &input.code, + &input.family, + &input.slug, + &input.message, + &input.remediation, + has_subject, + subject, + &input.implementation_fingerprint, + &input.config_json, + &input.dialect, + &input.project_dir, + ], + ) +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_subjects.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_subjects.rs new file mode 100644 index 0000000000..46616300a8 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/rule_subjects.rs @@ -0,0 +1,35 @@ +//! `rule_subjects`: the invocations one custom rule runs, with each subject's identity. + +use crate::custom_rule_cache::_helpers::digest::{field_digest, optional_fields}; +use crate::custom_rule_cache::constants::{ + MODEL_SUBJECT_KIND, PROJECT_SUBJECT, PROJECT_SUBJECT_KIND, PROJECT_SUBJECT_NAME, +}; +use crate::custom_rule_cache::models::{ModelSubjectInput, Subject, SubjectsInput}; + +/// The project subject for project-wide rules, otherwise every model sorted by path. +#[must_use] +pub fn rule_subjects(subject: Option<&str>, input: &SubjectsInput) -> Vec { + if subject == Some(PROJECT_SUBJECT_NAME) { + let [has_target, target] = optional_fields(input.target_name.as_deref()); + let count: String = input.models.len().to_string(); + return vec![Subject { + key: PROJECT_SUBJECT.to_owned(), + identity: field_digest(PROJECT_SUBJECT_KIND, &[&count, has_target, target]), + }]; + } + let mut models: Vec<&ModelSubjectInput> = input.models.iter().collect(); + models.sort_by(|left, right| left.path.cmp(&right.path)); + models + .into_iter() + .map(|model| { + let [has_materialized, materialized] = optional_fields(model.materialized.as_deref()); + Subject { + key: model.path.clone(), + identity: field_digest( + MODEL_SUBJECT_KIND, + &[&model.name, &model.path, has_materialized, materialized], + ), + } + }) + .collect() +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/store_custom_results.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/store_custom_results.rs new file mode 100644 index 0000000000..d313b2c242 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/store_custom_results.rs @@ -0,0 +1,67 @@ +//! `store_custom_results`: merge newly evaluated results into the stored rule caches. + +use std::collections::BTreeMap; + +use crate::custom_rule_cache::models::{CustomEvaluation, CustomRule, StoredEntry, StoredRule}; +use crate::custom_rule_cache::types::StoredRules; + +fn store_one( + mut current: StoredRule, + subject_identity: &str, + evaluation: &CustomEvaluation, +) -> StoredRule { + let Some((readset, Some(digest))) = &evaluation.cacheable else { + current.entries.remove(&evaluation.subject); + return current; + }; + let index: usize = match current.readsets.iter().position(|known| known == readset) { + Some(index) => index, + None => { + current.readsets.push(readset.clone()); + current.readsets.len() - 1 + } + }; + let entry = StoredEntry { + subject_identity: subject_identity.to_owned(), + readset: index, + reads_digest: digest.clone(), + findings: evaluation.findings.clone(), + }; + current.entries.insert(evaluation.subject.clone(), entry); + current +} + +/// Stored caches of the selected rules, with stale rule identities reset and new results stored. +#[must_use] +pub fn store_custom_results( + stored: &StoredRules, + rules: &[CustomRule], + evaluations: &[CustomEvaluation], +) -> StoredRules { + let mut updated = StoredRules::new(); + for rule in rules { + let Some(identity) = &rule.identity else { + continue; + }; + let mut current: StoredRule = stored + .get(&rule.code) + .filter(|previous| previous.identity == *identity) + .cloned() + .unwrap_or_else(|| StoredRule { + identity: identity.clone(), + ..StoredRule::default() + }); + let subjects: BTreeMap<&str, &str> = rule + .subjects + .iter() + .map(|subject| (subject.key.as_str(), subject.identity.as_str())) + .collect(); + for evaluation in evaluations.iter().filter(|item| item.code == rule.code) { + if let Some(subject_identity) = subjects.get(evaluation.subject.as_str()) { + current = store_one(current, subject_identity, evaluation); + } + } + updated.insert(rule.code.clone(), current); + } + updated +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/validate_custom_findings.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/validate_custom_findings.rs new file mode 100644 index 0000000000..de2fbf0025 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/validate_custom_findings.rs @@ -0,0 +1,61 @@ +//! `validate_custom_findings`: reject findings for other rules or unrelated model paths. + +use std::collections::BTreeSet; + +use serde_json::Value; + +use crate::custom_rule_cache::constants::{CODE_KEY, MODEL_SUBJECT_NAME, PATH_KEY}; +use crate::custom_rule_cache::errors::CustomFindingError; +use crate::custom_rule_cache::models::{CustomEvaluation, CustomRule, SubjectsInput}; + +fn shown(value: Option<&Value>) -> String { + match value { + Some(Value::String(text)) => text.clone(), + Some(other) => other.to_string(), + None => "None".to_owned(), + } +} + +/// An error for the first finding of an unselected rule or, for model rules, an unknown path. +pub fn validate_custom_findings( + rules: &[CustomRule], + project: &SubjectsInput, + evaluations: &[CustomEvaluation], +) -> Result<(), CustomFindingError> { + let codes: BTreeSet<&str> = rules.iter().map(|rule| rule.code.as_str()).collect(); + let model_rules: BTreeSet<&str> = rules + .iter() + .filter(|rule| rule.subject.as_deref() == Some(MODEL_SUBJECT_NAME)) + .map(|rule| rule.code.as_str()) + .collect(); + let paths: BTreeSet<&str> = project + .models + .iter() + .map(|model| model.path.as_str()) + .collect(); + for evaluation in evaluations { + for finding in &evaluation.findings { + let code: Option<&Value> = finding.get(CODE_KEY); + if !code + .and_then(Value::as_str) + .is_some_and(|code| codes.contains(code)) + { + return Err(CustomFindingError(format!( + "custom rule host returned a finding for unselected rule {}", + shown(code) + ))); + } + let path: Option<&Value> = finding.get(PATH_KEY); + let known: bool = path + .and_then(Value::as_str) + .is_some_and(|path| paths.contains(path)); + if model_rules.contains(evaluation.code.as_str()) && !known { + return Err(CustomFindingError(format!( + "model-subject custom rule returned a finding for unrelated path {}", + shown(path) + ))); + } + } + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/mod.rs b/crates/sqlbuild-rules/src/custom_rule_cache/mod.rs new file mode 100644 index 0000000000..6a67b87751 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/mod.rs @@ -0,0 +1,11 @@ +//! The custom-rule result cache: rule and subject identities, read-set lookup and the cache file. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/models.rs b/crates/sqlbuild-rules/src/custom_rule_cache/models.rs new file mode 100644 index 0000000000..d464c88238 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/models.rs @@ -0,0 +1,88 @@ +//! Custom-rule cache identity inputs, subjects, stored entries and evaluations. + +use std::collections::BTreeMap; + +use crate::custom_rule_cache::types::ReadSet; + +/// Everything besides recorded fact reads that changes one custom rule's results. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RuleIdentityInput { + pub code_identity: String, + pub runtime_version: String, + pub code: String, + pub family: String, + pub slug: String, + pub message: String, + pub remediation: String, + pub subject: Option, + pub implementation_fingerprint: String, + pub config_json: String, + pub dialect: String, + pub project_dir: String, +} + +/// One model as a custom-rule subject: path, name and materialization. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelSubjectInput { + pub path: String, + pub name: String, + pub materialized: Option, +} + +/// The project facts that decide which subjects custom rules run on. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SubjectsInput { + pub target_name: Option, + pub models: Vec, +} + +/// One rule invocation's subject: its key and an identity of the subject's own metadata. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Subject { + pub key: String, + pub identity: String, +} + +/// One selected custom rule, its identity when caching, subject kind and subjects. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CustomRule { + pub code: String, + pub identity: Option, + pub subject: Option, + pub subjects: Vec, +} + +/// One stored invocation result and the read set digest it was valid for. +#[derive(Clone, Debug, PartialEq)] +pub struct StoredEntry { + pub subject_identity: String, + pub readset: usize, + pub reads_digest: String, + pub findings: Vec, +} + +/// One rule's stored results, sharing read sets between subjects. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct StoredRule { + pub identity: String, + pub readsets: Vec, + pub entries: BTreeMap, +} + +/// Reused results, the subjects each rule must still evaluate (`None` for all) and counts. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CustomCacheLookup { + pub hits: Vec<((String, String), Vec)>, + pub plan: BTreeMap>>, + pub hit_count: usize, + pub miss_count: usize, +} + +/// One evaluated invocation, with its read set and digest when its result may be cached. +#[derive(Clone, Debug, PartialEq)] +pub struct CustomEvaluation { + pub code: String, + pub subject: String, + pub cacheable: Option<(ReadSet, Option)>, + pub findings: Vec, +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/tests.rs b/crates/sqlbuild-rules/src/custom_rule_cache/tests.rs new file mode 100644 index 0000000000..34532bb433 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_custom_rule_cache.rs"] +mod custom_rule_cache; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/tests/helpers.rs b/crates/sqlbuild-rules/src/custom_rule_cache/tests/helpers.rs new file mode 100644 index 0000000000..a90e902d58 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/tests/helpers.rs @@ -0,0 +1,53 @@ +use crate::custom_rule_cache::main::rule_subjects::rule_subjects; +use crate::custom_rule_cache::models::{ + CustomRule, ModelSubjectInput, RuleIdentityInput, SubjectsInput, +}; +use crate::custom_rule_cache::types::ReadSet; + +pub(crate) fn identity_input() -> RuleIdentityInput { + RuleIdentityInput { + code_identity: "code".to_owned(), + runtime_version: "runtime".to_owned(), + code: "CUSTOM001".to_owned(), + family: "CUSTOM".to_owned(), + slug: "orders-rule".to_owned(), + message: "message".to_owned(), + remediation: "remediation".to_owned(), + subject: Some("model".to_owned()), + implementation_fingerprint: "source-a".to_owned(), + config_json: "{}".to_owned(), + dialect: "duckdb".to_owned(), + project_dir: "/project".to_owned(), + } +} + +pub(crate) fn project(model_name: &str) -> SubjectsInput { + SubjectsInput { + target_name: Some("dev".to_owned()), + models: vec![ + ModelSubjectInput { + path: "models/orders.sql".to_owned(), + name: model_name.to_owned(), + materialized: None, + }, + ModelSubjectInput { + path: "models/customers.sql".to_owned(), + name: "customers".to_owned(), + materialized: Some("table".to_owned()), + }, + ], + } +} + +pub(crate) fn rule(model_name: &str) -> CustomRule { + CustomRule { + code: "CUSTOM001".to_owned(), + identity: Some("rule-identity".to_owned()), + subject: Some("model".to_owned()), + subjects: rule_subjects(Some("model"), &project(model_name)), + } +} + +pub(crate) fn readset() -> ReadSet { + vec![vec!["project.models".to_owned()]] +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs new file mode 100644 index 0000000000..807431d92f --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs @@ -0,0 +1,126 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::custom_rule_cache::main::custom_rule_cache_text::custom_rule_cache_text; +use crate::custom_rule_cache::main::lookup_custom_rule_cache::lookup_custom_rule_cache; +use crate::custom_rule_cache::main::parse_custom_rule_cache::parse_custom_rule_cache; +use crate::custom_rule_cache::main::pending_readsets::pending_readsets; +use crate::custom_rule_cache::main::rule_identity::rule_identity; +use crate::custom_rule_cache::main::store_custom_results::store_custom_results; +use crate::custom_rule_cache::models::{CustomEvaluation, CustomRule, RuleIdentityInput}; +use crate::custom_rule_cache::tests::helpers::{identity_input, readset, rule}; +use crate::custom_rule_cache::tests::test_types::{IdentityTestCase, LookupTestCase}; +use crate::custom_rule_cache::types::ReadSet; + +#[test] +fn given_rule_identity_inputs_when_one_changes_then_only_relevant_edits_change_the_identity() { + let test_cases = [ + IdentityTestCase { + description: "the same inputs give the same identity", + edit: |_| {}, + expected_changed: false, + }, + IdentityTestCase { + description: "an edited rule source changes the identity", + edit: |input| input.implementation_fingerprint = "source-b".to_owned(), + expected_changed: true, + }, + IdentityTestCase { + description: "edited rules configuration changes the identity", + edit: |input| input.config_json = "{\"select\":[]}".to_owned(), + expected_changed: true, + }, + IdentityTestCase { + description: "a project-wide subject changes the identity", + edit: |input| input.subject = Some("project".to_owned()), + expected_changed: true, + }, + ]; + + for test_case in test_cases { + let mut edited: RuleIdentityInput = identity_input(); + (test_case.edit)(&mut edited); + let changed: bool = rule_identity(&edited) != rule_identity(&identity_input()); + + assert_eq!( + changed, test_case.expected_changed, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_results_when_looking_up_then_unchanged_subjects_and_reads_hit() { + let test_cases = [ + LookupTestCase { + description: "unchanged reads and subjects hit every subject", + current_digest: "digest-a", + model_name: "orders", + expected_plan: vec![], + expected_hits: 2, + }, + LookupTestCase { + description: "a changed fact digest misses every subject", + current_digest: "digest-b", + model_name: "orders", + expected_plan: vec![("CUSTOM001", None)], + expected_hits: 0, + }, + LookupTestCase { + description: "a renamed model misses only that subject", + current_digest: "digest-a", + model_name: "orders_v2", + expected_plan: vec![("CUSTOM001", Some(vec!["models/orders.sql"]))], + expected_hits: 1, + }, + ]; + + for test_case in test_cases { + let stored_rule: CustomRule = rule("orders"); + let evaluations: Vec = stored_rule + .subjects + .iter() + .map(|subject| CustomEvaluation { + code: stored_rule.code.clone(), + subject: subject.key.clone(), + cacheable: Some((readset(), Some("digest-a".to_owned()))), + findings: vec![json!({"code": "CUSTOM001", "path": subject.key})], + }) + .collect(); + let text: String = custom_rule_cache_text( + &store_custom_results( + &BTreeMap::new(), + std::slice::from_ref(&stored_rule), + &evaluations, + ), + &[stored_rule], + ); + let current: CustomRule = rule(test_case.model_name); + let stored = parse_custom_rule_cache(&text); + let digests: BTreeMap = + pending_readsets(&stored, std::slice::from_ref(¤t)) + .into_iter() + .map(|readset| (readset, test_case.current_digest.to_owned())) + .collect(); + let lookup = lookup_custom_rule_cache(&stored, &[current], &digests); + let expected_plan: BTreeMap>> = test_case + .expected_plan + .iter() + .map(|(code, subjects)| { + let subjects = subjects + .as_ref() + .map(|items| items.iter().map(|item| (*item).to_owned()).collect()); + ((*code).to_owned(), subjects) + }) + .collect(); + + assert_eq!(lookup.plan, expected_plan, "{}", test_case.description); + assert_eq!( + lookup.hit_count, test_case.expected_hits, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_types.rs b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_types.rs new file mode 100644 index 0000000000..d09156ee8c --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_types.rs @@ -0,0 +1,13 @@ +pub(crate) struct IdentityTestCase { + pub(crate) description: &'static str, + pub(crate) edit: fn(&mut crate::custom_rule_cache::models::RuleIdentityInput), + pub(crate) expected_changed: bool, +} + +pub(crate) struct LookupTestCase { + pub(crate) description: &'static str, + pub(crate) current_digest: &'static str, + pub(crate) model_name: &'static str, + pub(crate) expected_plan: Vec<(&'static str, Option>)>, + pub(crate) expected_hits: usize, +} diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/types.rs b/crates/sqlbuild-rules/src/custom_rule_cache/types.rs new file mode 100644 index 0000000000..2c2458faa4 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_rule_cache/types.rs @@ -0,0 +1,14 @@ +//! Recorded fact reads and stored custom-rule caches. + +use std::collections::BTreeMap; + +use crate::custom_rule_cache::models::StoredRule; + +/// One compiler fact a custom rule read, as its key parts. +pub type FactKey = Vec; +/// The sorted, distinct facts one invocation read. +pub type ReadSet = Vec; +/// Every usable stored rule cache by rule code. +pub type StoredRules = BTreeMap; +/// Results by `(rule code, subject key)`, each a list of finding payloads. +pub type SubjectResults = BTreeMap<(String, String), Vec>; diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index 49dca7ea3c..11d313a627 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -4,10 +4,12 @@ pub mod configuration; pub mod constants; +pub mod custom_rule_cache; pub mod dsl_headers; pub mod engine; pub mod errors; pub mod models; +pub mod rule_cache; pub mod rule_results; pub mod rules; pub mod scope_metadata; diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 48117ae5f2..5b57a1a7d4 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -850,12 +850,6 @@ impl fensu_policy::policy::types::PolicyRule for RuleMetadata { } } -#[derive(Debug, Serialize)] -pub struct CatalogueResponse { - pub version: u32, - pub rules: Vec, -} - /// One compiled model's facts as the rules request builder receives them. #[derive(Clone, Debug, Default)] pub struct ModelRow { diff --git a/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs b/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs new file mode 100644 index 0000000000..50033d3c68 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs @@ -0,0 +1,19 @@ +//! SHA-256 over length-prefixed fields, so no two field lists share a key. + +use sha2::{Digest, Sha256}; + +use crate::constants::NATIVE_BUILD_IDENTITY; +use crate::rule_cache::constants::CACHE_VERSION; + +/// The hex digest of the cache version, the native build and each field in order. +pub(crate) fn field_digest(kind: &str, fields: &[&str]) -> String { + let mut digest = Sha256::new(); + for field in [CACHE_VERSION, NATIVE_BUILD_IDENTITY, kind] + .iter() + .chain(fields.iter()) + { + digest.update((field.len() as u64).to_le_bytes()); + digest.update(field.as_bytes()); + } + format!("{:x}", digest.finalize()) +} diff --git a/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs b/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs new file mode 100644 index 0000000000..71d61e1a07 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod digest; diff --git a/crates/sqlbuild-rules/src/rule_cache/constants.rs b/crates/sqlbuild-rules/src/rule_cache/constants.rs new file mode 100644 index 0000000000..4e3c6cd34a --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/constants.rs @@ -0,0 +1,15 @@ +//! The SQL rule result cache format and the inputs its keys separate. + +/// Bumped whenever key derivation or the stored finding shape changes. +pub(crate) const CACHE_VERSION: &str = "sql-rules-v6"; +pub(crate) const MODEL_KEY_KIND: &str = "model"; +pub(crate) const FILE_KEY_KIND: &str = "file"; +pub(crate) const LINT_KEY_KIND: &str = "lint"; +pub(crate) const NO_CONTRACT: &str = ""; +pub(crate) const MACRO_TOKEN: &str = "@"; +pub(crate) const TEMPLATE_INTERPOLATION_START: &str = "${"; +pub(crate) const VERSION_KEY: &str = "version"; +pub(crate) const BUILD_IDENTITY_KEY: &str = "build_identity"; +pub(crate) const ENTRIES_KEY: &str = "entries"; +pub(crate) const IDENTITY_KEY: &str = "identity"; +pub(crate) const FINDINGS_KEY: &str = "findings"; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/file_rule_key.rs b/crates/sqlbuild-rules/src/rule_cache/main/file_rule_key.rs new file mode 100644 index 0000000000..ffe137298a --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/file_rule_key.rs @@ -0,0 +1,23 @@ +//! `file_rule_key`: the cache key of one non-model SQL file's rule findings. + +use crate::rule_cache::_helpers::digest::field_digest; +use crate::rule_cache::constants::{FILE_KEY_KIND, MACRO_TOKEN, TEMPLATE_INTERPOLATION_START}; + +/// A digest of the lint key, codes, path and text; `None` when macros or templates expand it. +#[must_use] +pub fn file_rule_key( + lint_key: &str, + codes: &[String], + (relative_path, contents): (&str, &str), + expanded: bool, +) -> Option { + if expanded || contents.contains(MACRO_TOKEN) || contents.contains(TEMPLATE_INTERPOLATION_START) + { + return None; + } + let codes: String = codes.join("\0"); + Some(field_digest( + FILE_KEY_KIND, + &[lint_key, &codes, relative_path, contents], + )) +} diff --git a/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs b/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs new file mode 100644 index 0000000000..8f5b8bdc4d --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs @@ -0,0 +1,35 @@ +//! `lint_key`: the lint settings part of every SQL rule cache key. + +use crate::rule_cache::_helpers::digest::field_digest; +use crate::rule_cache::constants::LINT_KEY_KIND; +use crate::rule_cache::models::LintKeyInput; + +fn key_sets(keys: &[Vec]) -> String { + let mut sets: Vec = Vec::with_capacity(keys.len()); + for columns in keys { + sets.push(columns.join("\u{1f}")); + } + sets.join("\u{1d}") +} + +/// A digest of the dialect, literal and ranking limits, sorted relation keys and suppressions. +#[must_use] +pub fn lint_key(input: &LintKeyInput) -> String { + let mut relation_keys: Vec<(String, Vec>)> = input.relation_keys.clone(); + relation_keys.sort(); + let relations: String = relation_keys + .iter() + .map(|(relation, keys)| format!("{relation}\u{1e}{}", key_sets(keys))) + .collect::>() + .join("\u{1c}"); + field_digest( + LINT_KEY_KIND, + &[ + &input.dialect, + &input.max_literal_length.to_string(), + &input.max_ranking_order_by.to_string(), + &relations, + if input.allow_suppressions { "1" } else { "0" }, + ], + ) +} diff --git a/crates/sqlbuild-rules/src/rule_cache/main/lookup_rule_cache.rs b/crates/sqlbuild-rules/src/rule_cache/main/lookup_rule_cache.rs new file mode 100644 index 0000000000..2f70facb5c --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/lookup_rule_cache.rs @@ -0,0 +1,19 @@ +//! `lookup_rule_cache`: which paths' stored findings are still valid for their current keys. + +use crate::rule_cache::models::CacheLookup; +use crate::rule_cache::types::CachedEntries; + +/// Hits in key order of `keys`, with every path whose stored key differs or is absent missed. +#[must_use] +pub fn lookup_rule_cache(entries: &CachedEntries, keys: &[(String, String)]) -> CacheLookup { + let mut lookup = CacheLookup::default(); + for (path, key) in keys { + match entries.get(path) { + Some(entry) if entry.identity == *key => { + lookup.hits.push((path.clone(), entry.findings.clone())); + } + _ => lookup.misses.push(path.clone()), + } + } + lookup +} diff --git a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs new file mode 100644 index 0000000000..528348321d --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs @@ -0,0 +1,6 @@ +pub mod file_rule_key; +pub mod lint_key; +pub mod lookup_rule_cache; +pub mod model_rule_key; +pub mod parse_rule_cache; +pub mod rule_cache_text; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs b/crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs new file mode 100644 index 0000000000..99df4d0c9e --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs @@ -0,0 +1,24 @@ +//! `model_rule_key`: the cache key of one model's SQL rule findings. + +use crate::rule_cache::_helpers::digest::field_digest; +use crate::rule_cache::constants::{MODEL_KEY_KIND, NO_CONTRACT}; +use crate::rule_cache::models::ModelKeyInput; + +/// A digest of the lint key, selected codes, authored and compiled SQL and dynamic contract. +#[must_use] +pub fn model_rule_key(lint_key: &str, codes: &[String], model: &ModelKeyInput<'_>) -> String { + let codes: String = codes.join("\0"); + let contract: &str = model.contract.unwrap_or(NO_CONTRACT); + let has_contract: &str = if model.contract.is_some() { "1" } else { "0" }; + field_digest( + MODEL_KEY_KIND, + &[ + lint_key, + &codes, + model.authored_sql, + model.query_sql, + has_contract, + contract, + ], + ) +} diff --git a/crates/sqlbuild-rules/src/rule_cache/main/parse_rule_cache.rs b/crates/sqlbuild-rules/src/rule_cache/main/parse_rule_cache.rs new file mode 100644 index 0000000000..626fa2952d --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/parse_rule_cache.rs @@ -0,0 +1,35 @@ +//! `parse_rule_cache`: the usable entries of a stored SQL rule cache. + +use serde_json::{Map, Value}; + +use crate::constants::NATIVE_BUILD_IDENTITY; +use crate::rule_cache::constants::{ + BUILD_IDENTITY_KEY, CACHE_VERSION, ENTRIES_KEY, FINDINGS_KEY, IDENTITY_KEY, VERSION_KEY, +}; +use crate::rule_cache::models::CachedEntry; +use crate::rule_cache::types::CachedEntries; + +fn entry(value: &Value) -> Option { + let object: &Map = value.as_object()?; + Some(CachedEntry { + identity: object.get(IDENTITY_KEY)?.as_str()?.to_owned(), + findings: object.get(FINDINGS_KEY)?.as_array()?.clone(), + }) +} + +/// Entries of a cache written by this format and native build; anything else reads as empty. +#[must_use] +pub fn parse_rule_cache(text: &str) -> CachedEntries { + let Ok(Value::Object(payload)) = serde_json::from_str::(text) else { + return CachedEntries::new(); + }; + let current: bool = payload.get(VERSION_KEY).and_then(Value::as_str) == Some(CACHE_VERSION) + && payload.get(BUILD_IDENTITY_KEY).and_then(Value::as_str) == Some(NATIVE_BUILD_IDENTITY); + let Some(Value::Object(entries)) = payload.get(ENTRIES_KEY).filter(|_| current) else { + return CachedEntries::new(); + }; + entries + .iter() + .filter_map(|(path, value)| Some((path.clone(), entry(value)?))) + .collect() +} diff --git a/crates/sqlbuild-rules/src/rule_cache/main/rule_cache_text.rs b/crates/sqlbuild-rules/src/rule_cache/main/rule_cache_text.rs new file mode 100644 index 0000000000..0eabbd6065 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/rule_cache_text.rs @@ -0,0 +1,27 @@ +//! `rule_cache_text`: the stored form of the SQL rule cache. + +use serde_json::{Map, Value, json}; + +use crate::constants::NATIVE_BUILD_IDENTITY; +use crate::rule_cache::constants::{ + BUILD_IDENTITY_KEY, CACHE_VERSION, ENTRIES_KEY, FINDINGS_KEY, IDENTITY_KEY, VERSION_KEY, +}; +use crate::rule_cache::types::CachedEntries; + +/// Compact JSON of the format version, native build and every entry by path. +#[must_use] +pub fn rule_cache_text(entries: &CachedEntries) -> String { + let stored: Map = entries + .iter() + .map(|(path, entry)| { + let value = json!({ IDENTITY_KEY: entry.identity, FINDINGS_KEY: entry.findings }); + (path.clone(), value) + }) + .collect(); + json!({ + VERSION_KEY: CACHE_VERSION, + BUILD_IDENTITY_KEY: NATIVE_BUILD_IDENTITY, + ENTRIES_KEY: stored, + }) + .to_string() +} diff --git a/crates/sqlbuild-rules/src/rule_cache/mod.rs b/crates/sqlbuild-rules/src/rule_cache/mod.rs new file mode 100644 index 0000000000..76c842f679 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-rules/src/rule_cache/models.rs b/crates/sqlbuild-rules/src/rule_cache/models.rs new file mode 100644 index 0000000000..accfe26913 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/models.rs @@ -0,0 +1,35 @@ +//! SQL rule cache key inputs, stored entries and lookups. + +/// Everything besides SQL text that changes SQL rule findings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LintKeyInput { + pub dialect: String, + pub max_literal_length: i64, + pub max_ranking_order_by: i64, + /// Each relation and its key column sets, in any order. + pub relation_keys: Vec<(String, Vec>)>, + pub allow_suppressions: bool, +} + +/// One model's SQL rule inputs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelKeyInput<'a> { + pub authored_sql: &'a str, + pub query_sql: &'a str, + /// The model's dynamic column contract in a canonical text form, if it has one. + pub contract: Option<&'a str>, +} + +/// One stored entry: the key it was computed for and its findings as JSON values. +#[derive(Clone, Debug, PartialEq)] +pub struct CachedEntry { + pub identity: String, + pub findings: Vec, +} + +/// The stored findings of each path whose key still matches, and every other path. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CacheLookup { + pub hits: Vec<(String, Vec)>, + pub misses: Vec, +} diff --git a/crates/sqlbuild-rules/src/rule_cache/types.rs b/crates/sqlbuild-rules/src/rule_cache/types.rs new file mode 100644 index 0000000000..d2d2a48893 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/types.rs @@ -0,0 +1,8 @@ +//! The stored entries of one SQL rule cache. + +use std::collections::BTreeMap; + +use crate::rule_cache::models::CachedEntry; + +/// Every usable stored entry by project-relative path. +pub type CachedEntries = BTreeMap; diff --git a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs new file mode 100644 index 0000000000..f2ad474e0c --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs @@ -0,0 +1,53 @@ +//! `builtin_options`: the options each built-in rule declares. + +use crate::rules::models::{BuiltinOption, BuiltinOptionKind}; + +const fn boolean(name: &'static str, description: &'static str) -> BuiltinOption { + BuiltinOption { + kind: BuiltinOptionKind::Boolean, + name, + description, + } +} + +/// The declared options of the built-in rule `code`, in declaration order. +#[must_use] +pub fn builtin_options(code: &str) -> Vec { + match code { + "SQBRCONTRACT102" => vec![boolean( + "allow_numeric_indicators", + "Accept numeric is_/has_/can_ indicator columns.", + )], + "SQBRCONTRACT103" => vec![ + boolean( + "allow_date_for_at", + "Accept DATE for columns ending in _at.", + ), + boolean( + "allow_numeric_epoch", + "Accept numeric epoch values for timestamp-named columns.", + ), + boolean( + "allow_encoded_values", + "Accept string or variant encoded timestamp values.", + ), + ], + "SQBRCONTRACT104" => vec![ + boolean( + "allow_timestamps", + "Accept timestamp types for columns ending in _date.", + ), + boolean( + "allow_encoded_values", + "Accept string or variant encoded date values.", + ), + ], + "SQBRTEST203" => vec![BuiltinOption { + kind: BuiltinOptionKind::StringList, + name: "allowed_tests", + description: "Reviewed SQL test names that may assert only that empty inputs produce \ + no rows; listed tests count toward min_tests_per_model.", + }], + _ => Vec::new(), + } +} diff --git a/crates/sqlbuild-rules/src/rules/main/mod.rs b/crates/sqlbuild-rules/src/rules/main/mod.rs index bde8969bbb..a877872374 100644 --- a/crates/sqlbuild-rules/src/rules/main/mod.rs +++ b/crates/sqlbuild-rules/src/rules/main/mod.rs @@ -1,4 +1,5 @@ pub mod assemble_catalogue; +pub mod builtin_options; pub mod catalogue; pub mod evaluate; pub mod evaluate_project; @@ -6,4 +7,5 @@ pub mod fingerprint; pub mod resolve_threshold_overrides; pub mod select; pub mod selected_codes; +pub mod selected_sql_codes; pub mod sql_test_coverage; diff --git a/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs new file mode 100644 index 0000000000..c9778e61d1 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs @@ -0,0 +1,29 @@ +//! `selected_sql_codes`: the built-in SQL rule codes a `select`/`ignore` pair enables. + +use crate::constants::SQL_RULE_PREFIX; +use crate::rules::main::catalogue::catalogue; + +fn matches_any(code: &str, selectors: &[String]) -> bool { + for selector in selectors { + if code.starts_with(selector.as_str()) { + return true; + } + } + false +} + +/// Catalogue SQL rule codes matching a `select` prefix and no `ignore` prefix, in catalogue order. +#[must_use] +pub fn selected_sql_codes(select: &[String], ignore: &[String]) -> Vec { + let mut codes: Vec = Vec::new(); + for rule in catalogue() { + let code: String = rule.code; + if code.starts_with(SQL_RULE_PREFIX) + && matches_any(&code, select) + && !matches_any(&code, ignore) + { + codes.push(code); + } + } + codes +} diff --git a/crates/sqlbuild-rules/src/rules/models.rs b/crates/sqlbuild-rules/src/rules/models.rs index 969da8bdd7..0d9c6ec2a9 100644 --- a/crates/sqlbuild-rules/src/rules/models.rs +++ b/crates/sqlbuild-rules/src/rules/models.rs @@ -49,3 +49,20 @@ impl FaultCollector { self.faults.into_inner() } } + +/// The value kind of one built-in rule option. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BuiltinOptionKind { + /// A boolean defaulting to false. + Boolean, + /// A list of strings defaulting to empty. + StringList, +} + +/// One option a built-in rule declares. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BuiltinOption { + pub kind: BuiltinOptionKind, + pub name: &'static str, + pub description: &'static str, +} diff --git a/crates/sqlbuild-state/Cargo.toml b/crates/sqlbuild-state/Cargo.toml new file mode 100644 index 0000000000..b8436c7922 --- /dev/null +++ b/crates/sqlbuild-state/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "sqlbuild-state" +version.workspace = true +description = "Append-only warehouse state rows: event IDs, typed event SQL and source freshness versions." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +sha2.workspace = true +sqlbuild-core.workspace = true +sqlbuild-cursor.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-state/src/events/_helpers/identity.rs b/crates/sqlbuild-state/src/events/_helpers/identity.rs new file mode 100644 index 0000000000..3a9cf11060 --- /dev/null +++ b/crates/sqlbuild-state/src/events/_helpers/identity.rs @@ -0,0 +1,49 @@ +//! Content-addressed event IDs: SHA-256 of a compact, ASCII-escaped JSON identity array. + +use sha2::{Digest, Sha256}; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::events::constants::{COMPACT_ITEM_SEPARATOR, COMPACT_KEY_SEPARATOR}; +use crate::events::models::EventRelation; + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +pub(crate) fn optional_text(value: Option<&str>) -> JsonValue { + value.map_or(JsonValue::Null, text) +} + +/// `MigrationRelation.identity`: lowercased database or null, schema or `""`, and name. +pub(crate) fn relation_identity(relation: &EventRelation) -> JsonValue { + JsonValue::Array(vec![ + optional_text( + relation + .database + .as_deref() + .map(str::to_lowercase) + .as_deref(), + ), + text( + &relation + .schema + .as_deref() + .unwrap_or_default() + .to_lowercase(), + ), + text(&relation.name.to_lowercase()), + ]) +} + +/// `sha256(json.dumps(value, separators=(",", ":"), ensure_ascii=True))` as hex. +pub(crate) fn json_digest(value: &JsonValue, sort_keys: bool) -> Result { + let options = StdlibJsonOptions { + sort_keys, + ..StdlibJsonOptions::new(None) + .with_separators(COMPACT_ITEM_SEPARATOR, COMPACT_KEY_SEPARATOR) + }; + let encoded: String = dumps(value, &JsonDialect::Stdlib(options))?; + Ok(format!("{:x}", Sha256::digest(encoded.as_bytes()))) +} diff --git a/crates/sqlbuild-state/src/events/_helpers/literals.rs b/crates/sqlbuild-state/src/events/_helpers/literals.rs new file mode 100644 index 0000000000..f5ddca242e --- /dev/null +++ b/crates/sqlbuild-state/src/events/_helpers/literals.rs @@ -0,0 +1,46 @@ +//! Typed SQL literals for state event columns, as `render_state_sql_literal` writes them. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::events::constants::{ + ESCAPED_QUOTE, NULL_LITERAL, QUOTE, STRING_VALUE_REQUIRED, TIMESTAMP_VALUE_REQUIRED, UTC_SUFFIX, +}; +use crate::events::errors::StateSqlError; +use crate::events::models::{StateColumnType, StateValue}; + +/// `value` in single quotes with embedded quotes doubled. +pub(crate) fn quoted(value: &str) -> String { + format!("{QUOTE}{}{QUOTE}", value.replace(QUOTE, ESCAPED_QUOTE)) +} + +/// `datetime.isoformat()` of a UTC timestamp. +pub(crate) fn utc_isoformat(local: &LocalDateTime) -> String { + let date = &local.date; + let mut text: String = format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", + date.year, date.month, date.day, local.hour, local.minute, local.second + ); + if local.microsecond != 0 { + text.push_str(&format!(".{:06}", local.microsecond)); + } + text.push_str(UTC_SUFFIX); + text +} + +/// One value as the literal its declared column type takes. +pub(crate) fn literal( + column_type: StateColumnType, + value: &StateValue, +) -> Result { + match (column_type, value) { + (_, StateValue::Null) => Ok(NULL_LITERAL.to_owned()), + (StateColumnType::String, StateValue::Text(text)) => Ok(quoted(text)), + (StateColumnType::String, _) => Err(StateSqlError(STRING_VALUE_REQUIRED.to_owned())), + (StateColumnType::TextTimestamp, StateValue::UtcTimestamp(local)) => { + Ok(quoted(&utc_isoformat(local))) + } + (StateColumnType::TextTimestamp, _) => { + Err(StateSqlError(TIMESTAMP_VALUE_REQUIRED.to_owned())) + } + } +} diff --git a/crates/sqlbuild-state/src/events/_helpers/mod.rs b/crates/sqlbuild-state/src/events/_helpers/mod.rs new file mode 100644 index 0000000000..059db07285 --- /dev/null +++ b/crates/sqlbuild-state/src/events/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod identity; +pub(crate) mod literals; diff --git a/crates/sqlbuild-state/src/events/constants.rs b/crates/sqlbuild-state/src/events/constants.rs new file mode 100644 index 0000000000..e6992b279b --- /dev/null +++ b/crates/sqlbuild-state/src/events/constants.rs @@ -0,0 +1,12 @@ +//! Identity tags and messages of append-only migration state events. + +pub(crate) const COLUMN_EVENT_TAG: &str = "column"; +pub(crate) const OLD_NAME_EVENT_TAG: &str = "old_name"; +pub(crate) const COMPACT_ITEM_SEPARATOR: &str = ","; +pub(crate) const COMPACT_KEY_SEPARATOR: &str = ":"; +pub(crate) const QUOTE: char = '\''; +pub(crate) const ESCAPED_QUOTE: &str = "''"; +pub(crate) const NULL_LITERAL: &str = "NULL"; +pub(crate) const UTC_SUFFIX: &str = "+00:00"; +pub const STRING_VALUE_REQUIRED: &str = "state string columns require str values"; +pub const TIMESTAMP_VALUE_REQUIRED: &str = "state timestamp columns require datetime values"; diff --git a/crates/sqlbuild-state/src/events/errors.rs b/crates/sqlbuild-state/src/events/errors.rs new file mode 100644 index 0000000000..0c60fe83bc --- /dev/null +++ b/crates/sqlbuild-state/src/events/errors.rs @@ -0,0 +1,15 @@ +//! Failures rendering one state event row. + +use std::fmt::{self, Display}; + +/// A value whose kind does not fit its declared state column, with Python's message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StateSqlError(pub String); + +impl Display for StateSqlError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for StateSqlError {} diff --git a/crates/sqlbuild-state/src/events/main/column_migration_event_id.rs b/crates/sqlbuild-state/src/events/main/column_migration_event_id.rs new file mode 100644 index 0000000000..012ff3d7f2 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/column_migration_event_id.rs @@ -0,0 +1,25 @@ +//! `column_migration_event_id`: the content-addressed ID of one column rename by one run. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::events::_helpers::identity::{json_digest, optional_text, relation_identity, text}; +use crate::events::constants::COLUMN_EVENT_TAG; +use crate::events::models::EventRelation; + +/// SHA-256 of `["column", run_id, target_name, relation identity, origin, destination]`. +pub fn column_migration_event_id( + (run_id, target_name): (&str, Option<&str>), + relation: &EventRelation, + (origin_column, destination_column): (&str, &str), +) -> Result { + let identity = JsonValue::Array(vec![ + text(COLUMN_EVENT_TAG), + text(run_id), + optional_text(target_name), + relation_identity(relation), + text(&origin_column.to_lowercase()), + text(&destination_column.to_lowercase()), + ]); + json_digest(&identity, false) +} diff --git a/crates/sqlbuild-state/src/events/main/event_exists_sql.rs b/crates/sqlbuild-state/src/events/main/event_exists_sql.rs new file mode 100644 index 0000000000..2afc579fac --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/event_exists_sql.rs @@ -0,0 +1,12 @@ +//! `event_exists_sql`: the lookup that makes an append-only event write idempotent. + +use crate::events::_helpers::literals::quoted; + +/// `SELECT event_id FROM WHERE event_id = ''`. +#[must_use] +pub fn event_exists_sql(table: &str, event_id: &str) -> String { + format!( + "SELECT event_id FROM {table} WHERE event_id = {}", + quoted(event_id) + ) +} diff --git a/crates/sqlbuild-state/src/events/main/event_insert_sql.rs b/crates/sqlbuild-state/src/events/main/event_insert_sql.rs new file mode 100644 index 0000000000..bd82422553 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/event_insert_sql.rs @@ -0,0 +1,19 @@ +//! `event_insert_sql`: one append-only event row as `INSERT ... VALUES` with typed literals. + +use crate::events::_helpers::literals::literal; +use crate::events::errors::StateSqlError; +use crate::events::models::StateCell; + +/// `INSERT INTO
() VALUES ()`, failing on the first ill-typed value. +pub fn event_insert_sql(table: &str, cells: &[StateCell]) -> Result { + let literals: Vec = cells + .iter() + .map(|cell| literal(cell.column_type, &cell.value)) + .collect::>()?; + let columns: Vec<&str> = cells.iter().map(|cell| cell.column.as_str()).collect(); + Ok(format!( + "INSERT INTO {table} ({}) VALUES ({})", + columns.join(", "), + literals.join(", ") + )) +} diff --git a/crates/sqlbuild-state/src/events/main/migration_event_id.rs b/crates/sqlbuild-state/src/events/main/migration_event_id.rs new file mode 100644 index 0000000000..325d2fca7c --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/migration_event_id.rs @@ -0,0 +1,22 @@ +//! `migration_event_id`: the content-addressed ID of one relation move performed by one run. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::events::_helpers::identity::{json_digest, optional_text, relation_identity, text}; +use crate::events::models::EventRelation; + +/// SHA-256 of `[run_id, target_name, origin identity, destination identity]`. +pub fn migration_event_id( + (run_id, target_name): (&str, Option<&str>), + origin: &EventRelation, + destination: &EventRelation, +) -> Result { + let identity = JsonValue::Array(vec![ + text(run_id), + optional_text(target_name), + relation_identity(origin), + relation_identity(destination), + ]); + json_digest(&identity, false) +} diff --git a/crates/sqlbuild-state/src/events/main/mod.rs b/crates/sqlbuild-state/src/events/main/mod.rs new file mode 100644 index 0000000000..3870be1589 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/mod.rs @@ -0,0 +1,5 @@ +pub mod column_migration_event_id; +pub mod event_exists_sql; +pub mod event_insert_sql; +pub mod migration_event_id; +pub mod old_name_view_event_id; diff --git a/crates/sqlbuild-state/src/events/main/old_name_view_event_id.rs b/crates/sqlbuild-state/src/events/main/old_name_view_event_id.rs new file mode 100644 index 0000000000..785fd45008 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/old_name_view_event_id.rs @@ -0,0 +1,20 @@ +//! `old_name_view_event_id`: the content-addressed ID of one old-name step of a recorded move. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::events::_helpers::identity::{json_digest, text}; +use crate::events::constants::OLD_NAME_EVENT_TAG; + +/// SHA-256 of `["old_name", event_type, migration_event_id]`. +pub fn old_name_view_event_id( + event_type: &str, + migration_event_id: &str, +) -> Result { + let identity = JsonValue::Array(vec![ + text(OLD_NAME_EVENT_TAG), + text(event_type), + text(migration_event_id), + ]); + json_digest(&identity, false) +} diff --git a/crates/sqlbuild-state/src/events/mod.rs b/crates/sqlbuild-state/src/events/mod.rs new file mode 100644 index 0000000000..899237ebcb --- /dev/null +++ b/crates/sqlbuild-state/src/events/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/events/models.rs b/crates/sqlbuild-state/src/events/models.rs new file mode 100644 index 0000000000..9f850af603 --- /dev/null +++ b/crates/sqlbuild-state/src/events/models.rs @@ -0,0 +1,37 @@ +//! Relations, typed state values and column types of migration state events. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +/// A relation named by a migration event; only the name is required. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EventRelation { + pub database: Option, + pub schema: Option, + pub name: String, +} + +/// The declared physical type of a state event column. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StateColumnType { + String, + TextTimestamp, +} + +/// One value of an event row as the caller holds it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StateValue { + Null, + Text(String), + /// A timestamp already normalized to UTC wall time. + UtcTimestamp(LocalDateTime), + /// Any other value, which no supported column type accepts. + Other, +} + +/// One column of an event row: its name, declared type and value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StateCell { + pub column: String, + pub column_type: StateColumnType, + pub value: StateValue, +} diff --git a/crates/sqlbuild-state/src/events/tests.rs b/crates/sqlbuild-state/src/events/tests.rs new file mode 100644 index 0000000000..b55f12f730 --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_insert_sql.rs"] +mod insert_sql; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/events/tests/helpers.rs b/crates/sqlbuild-state/src/events/tests/helpers.rs new file mode 100644 index 0000000000..3ad5209a10 --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/helpers.rs @@ -0,0 +1,9 @@ +use crate::events::models::{StateCell, StateColumnType, StateValue}; + +pub(crate) fn cell(column: &str, column_type: StateColumnType, value: StateValue) -> StateCell { + StateCell { + column: column.to_owned(), + column_type, + value, + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_insert_sql.rs b/crates/sqlbuild-state/src/events/tests/test_insert_sql.rs new file mode 100644 index 0000000000..6da33933b0 --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/test_insert_sql.rs @@ -0,0 +1,47 @@ +use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; + +use crate::events::errors::StateSqlError; +use crate::events::main::event_insert_sql::event_insert_sql; +use crate::events::models::{StateColumnType, StateValue}; +use crate::events::tests::helpers::cell; +use crate::events::tests::test_types::InsertTestCase; + +#[test] +fn given_event_row_when_rendering_insert_then_matches_python_literals() { + let created = LocalDateTime { + date: CivilDate { + year: 2024, + month: 2, + day: 29, + }, + hour: 1, + minute: 2, + second: 3, + microsecond: 40, + }; + let test_cases = [ + InsertTestCase { + description: "quoted text, a null and a UTC timestamp", + cells: vec![ + cell("event_id", StateColumnType::String, StateValue::Text("o'k".to_owned())), + cell("target_name", StateColumnType::String, StateValue::Null), + cell("created_at", StateColumnType::TextTimestamp, StateValue::UtcTimestamp(created)), + ], + expected_sql: Ok("INSERT INTO s.t (event_id, target_name, created_at) VALUES ('o''k', NULL, '2024-02-29T01:02:03.000040+00:00')".to_owned()), + }, + InsertTestCase { + description: "a non-text value in a string column", + cells: vec![cell("event_id", StateColumnType::String, StateValue::Other)], + expected_sql: Err(StateSqlError("state string columns require str values".to_owned())), + }, + ]; + + for test_case in test_cases { + assert_eq!( + event_insert_sql("s.t", &test_case.cells), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_types.rs b/crates/sqlbuild-state/src/events/tests/test_types.rs new file mode 100644 index 0000000000..46f300b779 --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::events::errors::StateSqlError; +use crate::events::models::StateCell; + +pub(crate) struct InsertTestCase { + pub(crate) description: &'static str, + pub(crate) cells: Vec, + pub(crate) expected_sql: Result, +} diff --git a/crates/sqlbuild-state/src/freshness/_helpers/durations.rs b/crates/sqlbuild-state/src/freshness/_helpers/durations.rs new file mode 100644 index 0000000000..f792162144 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/_helpers/durations.rs @@ -0,0 +1,34 @@ +//! Single-unit minute, hour or day durations, as `Duration.parse` and `fixed_seconds` read them. + +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_cursor::algebra::models::DurationDigits; + +use crate::freshness::constants::{SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE}; + +/// Seconds of a duration; `None` inside `Some` when they exceed `u128`. +pub(crate) type DurationSeconds = Option; + +fn is_zero(digits: &str) -> bool { + digits.bytes().all(|byte| byte == b'0') +} + +/// The seconds of `text` when it names exactly one of minutes, hours or days. +pub(crate) fn single_unit_seconds(text: &str) -> Option { + let digits: DurationDigits = parse_duration(text)?; + let fixed: [(&str, u128); 3] = [ + (&digits.days, SECONDS_PER_DAY), + (&digits.hours, SECONDS_PER_HOUR), + (&digits.minutes, SECONDS_PER_MINUTE), + ]; + let calendar_or_seconds: bool = + !is_zero(&digits.years) || !is_zero(&digits.months) || !is_zero(&digits.seconds); + let mut used = fixed.iter().filter(|(amount, _)| !is_zero(amount)); + let (amount, unit) = used.next()?; + if calendar_or_seconds || used.next().is_some() { + return None; + } + let Ok(value) = amount.parse::() else { + return Some(None); + }; + Some(value.checked_mul(*unit)) +} diff --git a/crates/sqlbuild-state/src/freshness/_helpers/mod.rs b/crates/sqlbuild-state/src/freshness/_helpers/mod.rs new file mode 100644 index 0000000000..3fc35392a0 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod durations; +pub(crate) mod moments; diff --git a/crates/sqlbuild-state/src/freshness/_helpers/moments.rs b/crates/sqlbuild-state/src/freshness/_helpers/moments.rs new file mode 100644 index 0000000000..78aca6613a --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/_helpers/moments.rs @@ -0,0 +1,42 @@ +//! Microsecond distances between UTC wall times. + +use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; + +use crate::freshness::constants::{ + DAYS_PER_ERA, DAYS_PER_YEAR, ERA_YEARS, FEBRUARY, MARCH_BASED_MONTH_DAYS, MICROS_PER_SECOND, + MONTH_LENGTH_DIVISOR, MONTHS_FROM_MARCH, SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE, +}; + +/// Days since 0000-03-01 in the proleptic Gregorian calendar. +fn day_number(date: &CivilDate) -> i128 { + let shifted: bool = date.month <= FEBRUARY; + let year: i128 = i128::from(date.year) - i128::from(shifted); + let era: i128 = year.div_euclid(ERA_YEARS); + let year_of_era: i128 = year - era * ERA_YEARS; + let month: i128 = i128::from(date.month); + let month_from_march: i128 = if shifted { + month + MONTHS_FROM_MARCH + } else { + month - 3 + }; + let day_of_year: i128 = (MARCH_BASED_MONTH_DAYS * month_from_march + 2) / MONTH_LENGTH_DIVISOR + + i128::from(date.day) + - 1; + let day_of_era: i128 = + year_of_era * DAYS_PER_YEAR + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * DAYS_PER_ERA + day_of_era +} + +fn micros_of_day(local: &LocalDateTime) -> i128 { + let seconds: u128 = u128::from(local.hour) * SECONDS_PER_HOUR + + u128::from(local.minute) * SECONDS_PER_MINUTE + + u128::from(local.second); + (seconds * MICROS_PER_SECOND + u128::from(local.microsecond)) as i128 +} + +/// `later - earlier` in microseconds, as subtracting two `datetime` values gives it. +pub(crate) fn elapsed_micros(earlier: &LocalDateTime, later: &LocalDateTime) -> i128 { + let days: i128 = day_number(&later.date) - day_number(&earlier.date); + days * (SECONDS_PER_DAY * MICROS_PER_SECOND) as i128 + micros_of_day(later) + - micros_of_day(earlier) +} diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs new file mode 100644 index 0000000000..280fa1be44 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -0,0 +1,30 @@ +//! Value kinds, payload keys, duration units and messages of source freshness state. + +pub const TIMESTAMP_VALUE_KIND: &str = "timestamp"; +pub(crate) const SOURCE_NAME_KEY: &str = "source_name"; +pub(crate) const STRATEGY_KEY: &str = "strategy"; +pub(crate) const VALUE_KIND_KEY: &str = "value_kind"; +pub(crate) const DATA_VERSION_KEY: &str = "data_version"; +pub(crate) const COMPACT_ITEM_SEPARATOR: &str = ","; +pub(crate) const COMPACT_KEY_SEPARATOR: &str = ":"; +pub(crate) const UTC_SUFFIX: &str = "+00:00"; +pub(crate) const SECONDS_PER_MINUTE: u128 = 60; +pub(crate) const SECONDS_PER_HOUR: u128 = 3_600; +pub(crate) const SECONDS_PER_DAY: u128 = 86_400; +pub(crate) const MICROS_PER_SECOND: u128 = 1_000_000; +/// `timedelta.max.days`. +pub(crate) const MAX_TIMEDELTA_DAYS: u128 = 999_999_999; +/// The largest C `int` a `timedelta` day count passes through. +pub(crate) const MAX_C_INT: u128 = i32::MAX as u128; +pub(crate) const C_INT_OVERFLOW: &str = "Python int too large to convert to C int"; +pub const LAG_TOLERANCE_INVALID: &str = + "source freshness lag_tolerance must be a positive duration like 15m, 2h, or 1d"; +pub const AGE_POLICY_INVALID: &str = + "source freshness age_policy value must be a positive duration like 15m, 2h, or 1d"; +pub(crate) const ERA_YEARS: i128 = 400; +pub(crate) const DAYS_PER_ERA: i128 = 146_097; +pub(crate) const DAYS_PER_YEAR: i128 = 365; +pub(crate) const MARCH_BASED_MONTH_DAYS: i128 = 153; +pub(crate) const MONTH_LENGTH_DIVISOR: i128 = 5; +pub(crate) const MONTHS_FROM_MARCH: i128 = 9; +pub(crate) const FEBRUARY: u32 = 2; diff --git a/crates/sqlbuild-state/src/freshness/errors.rs b/crates/sqlbuild-state/src/freshness/errors.rs new file mode 100644 index 0000000000..b85513cbdb --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/errors.rs @@ -0,0 +1,24 @@ +//! Source freshness failures, each the exception Python raises for the same input. + +use std::fmt::{self, Display}; + +/// A failed freshness decision and the Python exception class that reports it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FreshnessError { + /// `SourceFreshnessObservationError`. + Observation(String), + /// `SourceFreshnessInputError`. + Input(String), + /// `OverflowError` from building a `timedelta`. + Overflow(String), +} + +impl Display for FreshnessError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Observation(message) | Self::Input(message) | Self::Overflow(message) => message, + }) + } +} + +impl std::error::Error for FreshnessError {} diff --git a/crates/sqlbuild-state/src/freshness/main/age_status.rs b/crates/sqlbuild-state/src/freshness/main/age_status.rs new file mode 100644 index 0000000000..7e51eef2b0 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/age_status.rs @@ -0,0 +1,41 @@ +//! `age_status`: an observed timestamp's age against its warn and error thresholds. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::freshness::_helpers::durations::single_unit_seconds; +use crate::freshness::_helpers::moments::elapsed_micros; +use crate::freshness::constants::{AGE_POLICY_INVALID, MICROS_PER_SECOND}; +use crate::freshness::errors::FreshnessError; +use crate::freshness::models::AgeStatus; + +/// Whether `age_seconds > threshold`, as Python compares a float with an integer. +fn older_than(age_micros: i128, threshold: &str) -> Result { + let Some(seconds) = single_unit_seconds(threshold) else { + return Err(FreshnessError::Input(AGE_POLICY_INVALID.to_owned())); + }; + let Some(seconds) = seconds else { + return Ok(false); + }; + let age_seconds: f64 = age_micros as f64 / MICROS_PER_SECOND as f64; + Ok(age_seconds > seconds as f64) +} + +/// The data version's age at `observed_at`: `error` past `error_after`, else `warn` past `warn_after`, else `pass`. +pub fn age_status( + (data_version, observed_at): (&LocalDateTime, &LocalDateTime), + error_after: Option<&str>, + warn_after: Option<&str>, +) -> Result { + let age_micros: i128 = elapsed_micros(data_version, observed_at); + if let Some(threshold) = error_after + && older_than(age_micros, threshold)? + { + return Ok(AgeStatus::Error); + } + if let Some(threshold) = warn_after + && older_than(age_micros, threshold)? + { + return Ok(AgeStatus::Warn); + } + Ok(AgeStatus::Pass) +} diff --git a/crates/sqlbuild-state/src/freshness/main/data_version_hash.rs b/crates/sqlbuild-state/src/freshness/main/data_version_hash.rs new file mode 100644 index 0000000000..4fd854c9aa --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/data_version_hash.rs @@ -0,0 +1,37 @@ +//! `data_version_hash`: the stable source freshness identity used as a graph input. + +use sha2::{Digest, Sha256}; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::freshness::constants::{ + COMPACT_ITEM_SEPARATOR, COMPACT_KEY_SEPARATOR, DATA_VERSION_KEY, SOURCE_NAME_KEY, STRATEGY_KEY, + VALUE_KIND_KEY, +}; + +/// SHA-256 of the compact, key-sorted JSON of the four identity fields. +pub fn data_version_hash( + (source_name, strategy): (&str, &str), + value_kind: &str, + data_version: &str, +) -> Result { + let payload = JsonValue::Object( + [ + (SOURCE_NAME_KEY, source_name), + (STRATEGY_KEY, strategy), + (VALUE_KIND_KEY, value_kind), + (DATA_VERSION_KEY, data_version), + ] + .into_iter() + .map(|(key, value)| (key.to_owned(), JsonValue::String(value.to_owned()))) + .collect(), + ); + let options = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None) + .with_separators(COMPACT_ITEM_SEPARATOR, COMPACT_KEY_SEPARATOR) + }; + let encoded: String = dumps(&payload, &JsonDialect::Stdlib(options))?; + Ok(format!("{:x}", Sha256::digest(encoded.as_bytes()))) +} diff --git a/crates/sqlbuild-state/src/freshness/main/mod.rs b/crates/sqlbuild-state/src/freshness/main/mod.rs new file mode 100644 index 0000000000..6e7cddd828 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/mod.rs @@ -0,0 +1,4 @@ +pub mod age_status; +pub mod data_version_hash; +pub mod normalized_data_version; +pub mod records_equivalent; diff --git a/crates/sqlbuild-state/src/freshness/main/normalized_data_version.rs b/crates/sqlbuild-state/src/freshness/main/normalized_data_version.rs new file mode 100644 index 0000000000..306ef41e8e --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/normalized_data_version.rs @@ -0,0 +1,25 @@ +//! `normalized_data_version`: a data version as stable state text. + +use crate::freshness::constants::UTC_SUFFIX; +use crate::freshness::models::DataVersion; + +/// A timestamp's `isoformat()`, an integer's digits, or the text itself. +#[must_use] +pub fn normalized_data_version(value: &DataVersion) -> String { + let (local, utc) = match value { + DataVersion::Timestamp { local, utc } => (local, *utc), + DataVersion::Integer(text) | DataVersion::Text(text) => return text.clone(), + }; + let date = &local.date; + let mut text: String = format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", + date.year, date.month, date.day, local.hour, local.minute, local.second + ); + if local.microsecond != 0 { + text.push_str(&format!(".{:06}", local.microsecond)); + } + if utc { + text.push_str(UTC_SUFFIX); + } + text +} diff --git a/crates/sqlbuild-state/src/freshness/main/records_equivalent.rs b/crates/sqlbuild-state/src/freshness/main/records_equivalent.rs new file mode 100644 index 0000000000..8b308c9da5 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/records_equivalent.rs @@ -0,0 +1,63 @@ +//! `records_equivalent`: whether two freshness records let a source be skipped. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::freshness::_helpers::durations::single_unit_seconds; +use crate::freshness::_helpers::moments::elapsed_micros; +use crate::freshness::constants::{ + C_INT_OVERFLOW, LAG_TOLERANCE_INVALID, MAX_C_INT, MAX_TIMEDELTA_DAYS, MICROS_PER_SECOND, + SECONDS_PER_DAY, TIMESTAMP_VALUE_KIND, +}; +use crate::freshness::errors::FreshnessError; +use crate::freshness::models::ComparableRecord; +use crate::freshness::types::TimestampParser; + +/// `timedelta(seconds=...)` of a lag tolerance in microseconds, failing as Python does. +fn tolerance_micros(text: &str) -> Result { + let Some(seconds) = single_unit_seconds(text) else { + return Err(FreshnessError::Observation( + LAG_TOLERANCE_INVALID.to_owned(), + )); + }; + let days: Option = seconds.map(|seconds| seconds / SECONDS_PER_DAY); + match (seconds, days) { + (Some(seconds), Some(days)) if days <= MAX_TIMEDELTA_DAYS => { + Ok(seconds * MICROS_PER_SECOND) + } + (_, Some(days)) if days <= MAX_C_INT => Err(FreshnessError::Overflow(format!( + "days={days}; must have magnitude <= {MAX_TIMEDELTA_DAYS}" + ))), + _ => Err(FreshnessError::Overflow(C_INT_OVERFLOW.to_owned())), + } +} + +/// Equal hashes, or equal timestamps, or a later timestamp within `lag_tolerance`. +pub fn records_equivalent( + (previous, current): (&ComparableRecord, &ComparableRecord), + lag_tolerance: Option<&str>, + parser: &dyn TimestampParser, +) -> Result { + if previous.data_version_hash == current.data_version_hash { + return Ok(true); + } + if previous.value_kind != TIMESTAMP_VALUE_KIND || current.value_kind != TIMESTAMP_VALUE_KIND { + return Ok(false); + } + let (Some(previous_text), Some(current_text)) = (&previous.data_version, ¤t.data_version) + else { + return Ok(false); + }; + let previous_time: LocalDateTime = parser.utc_timestamp(previous_text)?; + let current_time: LocalDateTime = parser.utc_timestamp(current_text)?; + let elapsed: i128 = elapsed_micros(&previous_time, ¤t_time); + if elapsed == 0 { + return Ok(true); + } + let Some(tolerance) = lag_tolerance else { + return Ok(false); + }; + if elapsed < 0 { + return Ok(false); + } + Ok(elapsed.unsigned_abs() <= tolerance_micros(tolerance)?) +} diff --git a/crates/sqlbuild-state/src/freshness/mod.rs b/crates/sqlbuild-state/src/freshness/mod.rs new file mode 100644 index 0000000000..65aaca2009 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/freshness/models.rs b/crates/sqlbuild-state/src/freshness/models.rs new file mode 100644 index 0000000000..99ca646dfe --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/models.rs @@ -0,0 +1,32 @@ +//! Observed data versions, comparable state records and age statuses. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +/// An observed data version of one value kind. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DataVersion { + /// A timestamp's wall time; `utc` when it was aware and is now in UTC. + Timestamp { + local: LocalDateTime, + utc: bool, + }, + /// An integer's decimal text. + Integer(String), + Text(String), +} + +/// The fields of a stored or observed record that skip decisions compare. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ComparableRecord { + pub data_version_hash: String, + pub value_kind: String, + pub data_version: Option, +} + +/// How an observed timestamp's age compares with its policy thresholds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AgeStatus { + Pass, + Warn, + Error, +} diff --git a/crates/sqlbuild-state/src/freshness/tests.rs b/crates/sqlbuild-state/src/freshness/tests.rs new file mode 100644 index 0000000000..ead02b3221 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_equivalence.rs"] +mod equivalence; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/freshness/tests/helpers.rs b/crates/sqlbuild-state/src/freshness/tests/helpers.rs new file mode 100644 index 0000000000..ac0bb6bd79 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/tests/helpers.rs @@ -0,0 +1,31 @@ +use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; + +use crate::freshness::errors::FreshnessError; +use crate::freshness::models::ComparableRecord; +use crate::freshness::tests::test_types::MinuteParser; +use crate::freshness::types::TimestampParser; + +pub(crate) fn record(data_version: &str) -> ComparableRecord { + ComparableRecord { + data_version_hash: data_version.to_owned(), + value_kind: "timestamp".to_owned(), + data_version: Some(data_version.to_owned()), + } +} + +impl TimestampParser for MinuteParser { + fn utc_timestamp(&self, text: &str) -> Result { + let field = |range: std::ops::Range| text[range].parse::().unwrap_or(0); + Ok(LocalDateTime { + date: CivilDate { + year: i64::from(field(0..4)), + month: field(5..7), + day: field(8..10), + }, + hour: field(11..13), + minute: field(14..16), + second: 0, + microsecond: 0, + }) + } +} diff --git a/crates/sqlbuild-state/src/freshness/tests/test_equivalence.rs b/crates/sqlbuild-state/src/freshness/tests/test_equivalence.rs new file mode 100644 index 0000000000..c78af54e91 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/tests/test_equivalence.rs @@ -0,0 +1,56 @@ +use crate::freshness::errors::FreshnessError; +use crate::freshness::main::records_equivalent::records_equivalent; +use crate::freshness::tests::helpers::record; +use crate::freshness::tests::test_types::{EquivalenceTestCase, MinuteParser}; + +#[test] +fn given_timestamp_records_when_comparing_then_lag_tolerance_decides() { + let test_cases = [ + EquivalenceTestCase { + description: "later within a two-hour tolerance across a month end", + previous: "2024-01-31T23:30", + current: "2024-02-01T01:30", + lag_tolerance: Some("2h"), + expected_equivalent: Ok(true), + }, + EquivalenceTestCase { + description: "later beyond a fifteen-minute tolerance", + previous: "2024-01-31T23:30", + current: "2024-01-31T23:46", + lag_tolerance: Some("15m"), + expected_equivalent: Ok(false), + }, + EquivalenceTestCase { + description: "an earlier value is never equivalent", + previous: "2024-01-31T23:30", + current: "2024-01-31T23:29", + lag_tolerance: Some("1d"), + expected_equivalent: Ok(false), + }, + EquivalenceTestCase { + description: "a calendar tolerance is rejected", + previous: "2024-01-31T23:30", + current: "2024-01-31T23:31", + lag_tolerance: Some("1mo"), + expected_equivalent: Err(FreshnessError::Observation( + "source freshness lag_tolerance must be a positive duration like 15m, 2h, or 1d" + .to_owned(), + )), + }, + ]; + + for test_case in test_cases { + let (previous, current) = (record(test_case.previous), record(test_case.current)); + + assert_eq!( + records_equivalent( + (&previous, ¤t), + test_case.lag_tolerance, + &MinuteParser + ), + test_case.expected_equivalent, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/freshness/tests/test_types.rs b/crates/sqlbuild-state/src/freshness/tests/test_types.rs new file mode 100644 index 0000000000..0632d8083a --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/tests/test_types.rs @@ -0,0 +1,12 @@ +use crate::freshness::errors::FreshnessError; + +/// Parses `YYYY-MM-DDTHH:MM` text the tests write. +pub(crate) struct MinuteParser; + +pub(crate) struct EquivalenceTestCase { + pub(crate) description: &'static str, + pub(crate) previous: &'static str, + pub(crate) current: &'static str, + pub(crate) lag_tolerance: Option<&'static str>, + pub(crate) expected_equivalent: Result, +} diff --git a/crates/sqlbuild-state/src/freshness/types.rs b/crates/sqlbuild-state/src/freshness/types.rs new file mode 100644 index 0000000000..ed64105016 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/types.rs @@ -0,0 +1,11 @@ +//! The timestamp parsing a caller supplies for stored data versions. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::freshness::errors::FreshnessError; + +/// Parses stored timestamp text into UTC wall time. +pub trait TimestampParser { + /// `datetime.fromisoformat(text)` in UTC, naive text read as UTC. + fn utc_timestamp(&self, text: &str) -> Result; +} diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs new file mode 100644 index 0000000000..d62c133a7e --- /dev/null +++ b/crates/sqlbuild-state/src/lib.rs @@ -0,0 +1,6 @@ +//! Warehouse state rows SQLBuild appends: event IDs, event SQL and source freshness versions. + +#![forbid(unsafe_code)] + +pub mod events; +pub mod freshness; diff --git a/fensu.toml b/fensu.toml index 046d307047..bec5e7a491 100644 --- a/fensu.toml +++ b/fensu.toml @@ -369,6 +369,7 @@ crate_names = [ "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", + "sqlbuild-state", ] [[dupes.contract_exemptions]] diff --git a/python-boundary.toml b/python-boundary.toml index 253fe17c20..37c11ad33e 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 89820 +remaining_lines = 79435 [[stays]] path = "executor" @@ -59,11 +59,66 @@ path = "rules" reason = "public-api" note = "Public custom-rule API and test helper." +[[stays]] +path = "diagnostics/_helpers/logging.py" +reason = "public-api" +note = "Installs SQLBuild's handlers on Python's logging, which users and integrations configure." + +[[stays]] +path = "diagnostics/_helpers/logging_formatters.py" +reason = "public-api" +note = "Python logging formatters for console and file diagnostics." + +[[stays]] +path = "diagnostics/_helpers/redaction.py" +reason = "public-api" +note = "Collects redacted extra fields from a Python LogRecord; the redaction itself is native." + +[[stays]] +path = "diagnostics/classes/diagnostic_record_redactor.py" +reason = "public-api" +note = "Redaction entry points that Python logging handlers call per record." + +[[stays]] +path = "diagnostics/classes/diagnostics_console_formatter.py" +reason = "public-api" +note = "A logging.Formatter subclass for the console stream." + +[[stays]] +path = "diagnostics/classes/dynamic_stderr_handler.py" +reason = "public-api" +note = "A logging.Handler that follows the current sys.stderr." + +[[stays]] +path = "diagnostics/classes/invocation_diagnostic_routing.py" +reason = "public-api" +note = "Attaches and removes per-invocation logging handlers." + +[[stays]] +path = "diagnostics/classes/log_record_route_filter.py" +reason = "public-api" +note = "A logging.Filter that routes records to destinations." + [[stays]] path = "rule_engine/_helpers/host" reason = "user-code" note = "Subprocess host that loads and runs user rule modules." +[[stays]] +path = "rule_engine/_helpers/engine/custom_dispatch.py" +reason = "user-code" +note = "Submits custom rules to their worker thread beside native built-in rule evaluation." + +[[stays]] +path = "rule_engine/_helpers/engine/custom_rules.py" +reason = "user-code" +note = "Dispatches custom rules to the rule-host worker pool and digests the fact views they read." + +[[stays]] +path = "rule_engine/_helpers/run/rule_threads.py" +reason = "user-code" +note = "Runs rule phases and early SQL lint on worker threads with futures and stop events." + [[stays]] path = "rule_engine/classes/rule_context.py" reason = "user-code" @@ -235,10 +290,6 @@ note = "A test file that no longer parses is left unformatted, as before; no Pyt location = "lint/_helpers/native_sql.py::_native_responses" note = "A native lint failure is reported as a lint error result, not recomputed in Python." -[[handled_native_errors]] -location = "rule_engine/_helpers/run/rules.py::_cached_sql_findings" -note = "An undecodable cached finding is a cache miss; the rules run again natively." - [[handled_native_errors]] location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." diff --git a/scripts/native_module_paths/__init__.py b/scripts/native_module_paths/__init__.py new file mode 100644 index 0000000000..64eb0e51a9 --- /dev/null +++ b/scripts/native_module_paths/__init__.py @@ -0,0 +1 @@ +"""Native Rust-to-Python module path checks.""" diff --git a/scripts/native_module_paths/_helpers/__init__.py b/scripts/native_module_paths/_helpers/__init__.py new file mode 100644 index 0000000000..3f583f245c --- /dev/null +++ b/scripts/native_module_paths/_helpers/__init__.py @@ -0,0 +1 @@ +"""Native module path helpers.""" diff --git a/scripts/native_module_paths/_helpers/crates.py b/scripts/native_module_paths/_helpers/crates.py new file mode 100644 index 0000000000..00a9210d22 --- /dev/null +++ b/scripts/native_module_paths/_helpers/crates.py @@ -0,0 +1,40 @@ +"""Check every Python module path the native crates import by string.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.native_module_paths._helpers.resolve import unresolved_reference_error +from scripts.native_module_paths._helpers.scan import scan_native_source +from scripts.native_module_paths.constants import RUST_SOURCE_GLOB, SCANNED_SOURCE_ROOTS +from scripts.native_module_paths.models import ModulePathReference, NativeModulePathScan + + +def scan_native_crates(*, root: Path) -> NativeModulePathScan: + """Scan every Rust file of the crates that call Python, in path order.""" + references: list[ModulePathReference] = [] + logger_names: set[str] = set() + violations: list[str] = [] + for source_root in SCANNED_SOURCE_ROOTS: + for path in sorted((root / source_root).rglob(RUST_SOURCE_GLOB)): + scan: NativeModulePathScan = scan_native_source( + source=path.read_text(encoding="utf-8"), label=path.relative_to(root).as_posix() + ) + references.extend(scan.references) + logger_names.update(scan.logger_names) + violations.extend(scan.violations) + return NativeModulePathScan( + references=tuple(references), + logger_names=tuple(sorted(logger_names)), + violations=tuple(violations), + ) + + +def native_module_path_errors(*, scan: NativeModulePathScan) -> tuple[str, ...]: + """Every policy violation, then every reference that does not resolve.""" + unresolved: list[str] = [] + for reference in scan.references: + error: str | None = unresolved_reference_error(reference=reference) + if error is not None: + unresolved.append(error) + return scan.violations + tuple(unresolved) diff --git a/scripts/native_module_paths/_helpers/resolve.py b/scripts/native_module_paths/_helpers/resolve.py new file mode 100644 index 0000000000..0f9cc6619b --- /dev/null +++ b/scripts/native_module_paths/_helpers/resolve.py @@ -0,0 +1,33 @@ +"""Resolve module paths found in native source against the importable Python package.""" + +from __future__ import annotations + +import importlib + +from scripts.native_module_paths.models import ModulePathReference + + +def unresolved_reference_error(*, reference: ModulePathReference) -> str | None: + """Why the reference does not resolve to a module, module attribute or class member.""" + target: object | None = _resolved(path=reference.path) + if target is None: + return f"{reference.location}: '{reference.path}' is not an importable module or attribute." + if reference.attribute is not None and not hasattr(target, reference.attribute): + return f"{reference.location}: '{reference.path}' has no attribute '{reference.attribute}'." + return None + + +def _resolved(*, path: str) -> object | None: + """The longest importable module prefix of `path`, then its remaining attributes.""" + parts: list[str] = path.split(".") + for split in range(len(parts), 0, -1): + try: + target: object = importlib.import_module(".".join(parts[:split])) + except ImportError: + continue + for attribute in parts[split:]: + if not hasattr(target, attribute): + return None + target = getattr(target, attribute) + return target + return None diff --git a/scripts/native_module_paths/_helpers/scan.py b/scripts/native_module_paths/_helpers/scan.py new file mode 100644 index 0000000000..e830a6fc39 --- /dev/null +++ b/scripts/native_module_paths/_helpers/scan.py @@ -0,0 +1,76 @@ +"""Find the Python module paths native Rust source imports by string.""" + +from __future__ import annotations + +from scripts.native_module_paths.constants import ( + DOTTED_PATH, + DYNAMIC_MODULE_PATHS, + IMPORT_GETATTR_CHAIN, + IMPORT_HELPER_CALL, + LITERAL_MODULE_ATTRIBUTE_PAIR, + LOGGER_CONSTANT_NAME, + PACKAGE_LITERAL, + STRING_CONSTANT, +) +from scripts.native_module_paths.models import ModulePathReference, NativeModulePathScan + + +def scan_native_source(*, source: str, label: str) -> NativeModulePathScan: + """Return the module paths, module attributes and dynamic module paths in one Rust file.""" + constants: dict[str, str] = { + match.group(1): match.group(2) for match in STRING_CONSTANT.finditer(source) + } + logger_names: frozenset[str] = frozenset( + value for name, value in constants.items() if LOGGER_CONSTANT_NAME.search(name) + ) + references: list[ModulePathReference] = [] + violations: list[str] = [] + for match in PACKAGE_LITERAL.finditer(source): + path: str = match.group(1) + location: str = _location(source=source, label=label, offset=match.start()) + if DOTTED_PATH.fullmatch(path) is None: + violations.append(f"{location}: '{path}' is not a complete dotted module path.") + elif path not in logger_names: + references.append(ModulePathReference(location=location, path=path)) + for pattern in (IMPORT_HELPER_CALL, IMPORT_GETATTR_CHAIN): + for match in pattern.finditer(source): + module: str | None = _module_argument(argument=match.group(1), constants=constants) + if module is not None and module not in logger_names: + references.append( + ModulePathReference( + location=_location(source=source, label=label, offset=match.start()), + path=module, + attribute=match.group(2), + ) + ) + for match in LITERAL_MODULE_ATTRIBUTE_PAIR.finditer(source): + references.append( + ModulePathReference( + location=_location(source=source, label=label, offset=match.start()), + path=match.group(1), + attribute=match.group(2), + ) + ) + for pattern in DYNAMIC_MODULE_PATHS: + violations.extend( + f"{_location(source=source, label=label, offset=match.start())}: module path is " + "built at runtime; use a complete literal module path." + for match in pattern.finditer(source) + ) + return NativeModulePathScan( + references=tuple(references), + logger_names=tuple(sorted(logger_names)), + violations=tuple(violations), + ) + + +def _module_argument(*, argument: str, constants: dict[str, str]) -> str | None: + """The module a helper argument names: a literal, or a same-file package constant.""" + if argument.startswith('"'): + return argument.strip('"') + return constants.get(argument) + + +def _location(*, source: str, label: str, offset: int) -> str: + line: int = source.count("\n", 0, offset) + 1 + return f"{label}:{line}" diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py new file mode 100644 index 0000000000..7cf304ae84 --- /dev/null +++ b/scripts/native_module_paths/constants.py @@ -0,0 +1,34 @@ +"""Native module path policy constants; crates/README.md describes the rules.""" + +import re + +SCANNED_SOURCE_ROOTS: tuple[str, ...] = ("crates/sqlbuild-python/src", "crates/sqlbuild-cli/src") +RUST_SOURCE_GLOB: str = "*.rs" +PACKAGE_LITERAL: re.Pattern[str] = re.compile(r'"(sqlbuild\.[^"\\]*)"') +DOTTED_PATH: re.Pattern[str] = re.compile(r"sqlbuild(?:\.[A-Za-z_][A-Za-z0-9_]*)+") +STRING_CONSTANT: re.Pattern[str] = re.compile( + r"\bconst\s+([A-Z][A-Z0-9_]*)\s*:\s*&(?:'static\s+)?str\s*=\s*\"(sqlbuild\.[^\"\\]*)\"\s*;" +) +LOGGER_CONSTANT_NAME: re.Pattern[str] = re.compile(r"LOGGER") +ATTRIBUTE_NAME: str = r"\"([A-Za-z_][A-Za-z0-9_]*)\"" +MODULE_ARGUMENT: str = r"([A-Z][A-Z0-9_]*|\"sqlbuild\.[^\"\\]*\")" +IMPORT_HELPER_CALL: re.Pattern[str] = re.compile( + r"\b[a-z_][a-z0-9_]*\(\s*py\s*,\s*" + + MODULE_ARGUMENT + + r"\s*,\s*" + + ATTRIBUTE_NAME + + r"\s*,?\s*\)" +) +IMPORT_GETATTR_CHAIN: re.Pattern[str] = re.compile( + r"\bimport\(\s*" + MODULE_ARGUMENT + r"\s*\)\?\s*\.getattr\(\s*" + ATTRIBUTE_NAME + r"\s*\)" +) +LITERAL_MODULE_ATTRIBUTE_PAIR: re.Pattern[str] = re.compile( + r"\(\s*\"(sqlbuild\.[^\"\\]*)\"\s*,\s*" + ATTRIBUTE_NAME + r"\s*,?\s*\)" +) +DYNAMIC_MODULE_PATHS: tuple[re.Pattern[str], ...] = ( + re.compile(r"\b(?:format|concat)!\(\s*\"\{[A-Z0-9_]*(?:PACKAGE|MODULE)[A-Z0-9_]*\}\."), + re.compile( + r"\b(?:format|concat)!\(\s*\"\{\}\.[^\"]*\"\s*,\s*[A-Z0-9_]*(?:PACKAGE|MODULE)[A-Z0-9_]*\b" + ), + re.compile(r"\b(?:format|concat)!\(\s*\"sqlbuild\."), +) diff --git a/scripts/native_module_paths/models.py b/scripts/native_module_paths/models.py new file mode 100644 index 0000000000..9b20922809 --- /dev/null +++ b/scripts/native_module_paths/models.py @@ -0,0 +1,23 @@ +"""Native module path scan models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ModulePathReference: + """One Python module path, or module and attribute, that native source imports.""" + + location: str + path: str + attribute: str | None = None + + +@dataclass(frozen=True) +class NativeModulePathScan: + """What a scan of native source found: checked references and policy violations.""" + + references: tuple[ModulePathReference, ...] + logger_names: tuple[str, ...] + violations: tuple[str, ...] diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b92c6231d8..6e0b59ac04 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -17,13 +17,21 @@ from sqlbuild.adapter.contract.types import ( TypeDialect, ) from sqlbuild.adapter.relations.models import InspectionQueryRecord +from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.models import ( + CheckCommandRequest, + CloneCommandRequest, + DiffCommandRequest, FreshnessCommandRequest, JanitorCommandRequest, LineageCommandRequest, + LoadCommandRequest, PlaygroundCommandRequest, + QueryDiffPreparation, RefactorCommandRequest, ScopeCommandRequest, + SeedCommandRequest, + TestCommandRequest, ) from sqlbuild.cli.commands.types import CompileLineageMode, ScopeIndexLoader from sqlbuild.cli.compile.models import CompileAnalysis @@ -35,6 +43,7 @@ from sqlbuild.cli.entry.models import ( ResolvedWarehouse, ) from sqlbuild.cli.output.models import SkillMaintenanceResult, SkillUpdateResult, WrittenTarget +from sqlbuild.cli.progress.models import NestedProgressChildRow from sqlbuild.compiler.auditing.models import ( AuditGateIdentity, MeasurementContract, @@ -107,6 +116,7 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredAdapterFile, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredCheckFunction, DiscoveredConstantFile, DiscoveredEnumFile, DiscoveredHookFunction, @@ -127,6 +137,7 @@ from sqlbuild.compiler.discovery.models import ( DiscoveryFileFault, EnumDeclaration, ModelSchemaDeclaration, + SqlTestParameterDeclaration, TolerantScopeDiscovery, ) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet @@ -139,6 +150,8 @@ from sqlbuild.compiler.lineage.models import ( ModelColumnLineage, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability +from sqlbuild.compiler.migrations.models import MigrationRelation +from sqlbuild.compiler.migrations.types import OldNameViewEventType from sqlbuild.compiler.pipeline.models import ( CompiledProjectPhaseResult, CompilePipelineOptions, @@ -154,6 +167,7 @@ from sqlbuild.compiler.planner.models import ( FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, PlanOutput, + SqlTestPlanEntry, ) from sqlbuild.compiler.planner.types import CursorGrain from sqlbuild.compiler.python_nodes.models import ( @@ -193,6 +207,10 @@ from sqlbuild.compiler.scopes.models import ( ) from sqlbuild.compiler.scopes.types import JsonValue, ScopeQueryTarget from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord +from sqlbuild.compiler.source_freshness.types import ( + SourceFreshnessAgeStatus, + SourceFreshnessComparableRecord, +) from sqlbuild.compiler.sql_analysis.models import ( SqlBindingResult, SqlLexicalSyntax, @@ -211,6 +229,7 @@ from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult +from sqlbuild.executor.diff.models import DiffExecutionResult from sqlbuild.executor.janitor.models import JanitorExecutionResult, JanitorPlan from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes.models import ( @@ -219,6 +238,16 @@ from sqlbuild.executor.python_nodes.models import ( ) from sqlbuild.executor.scenario.models import ScenarioRunResult, ScenarioSnapshotCaptureRunResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult +from sqlbuild.integrations.dbt.models import ( + DbtInteropPlan, + DbtInteropSelectionResult, + DbtLsNode, + DbtManifestIndex, + DbtManifestModel, + NormalizedDbtProfileConnection, + ResolvedDbtProfileOutput, +) +from sqlbuild.integrations.dbt.types import DbtInteropCommand from sqlbuild.lint.models import HeaderSpan, LintConfig, LintViolation from sqlbuild.lint.types import RelationKeyRow from sqlbuild.presentation.classes.cli_style import CliStyle @@ -226,7 +255,8 @@ from sqlbuild.presentation.classes.transient_status_reporter import TransientSta from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.rule_engine.models import Finding, Rule, RulesResult +from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult +from sqlbuild.rule_engine.types import RuleCheck from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import ( @@ -241,11 +271,17 @@ from sqlbuild.spec.contracts.models import ( SchemaSeedEntry, SettingsConfig, SourceEntry, + SourceFreshnessAgePolicy, SourceLocation, TargetConfig, ) -from sqlbuild.spec.contracts.types import WarehouseGroup +from sqlbuild.spec.contracts.types import ( + SourceFreshnessStrategy, + SourceFreshnessValueKind, + WarehouseGroup, +) from sqlbuild.sql_values.models import SqlValue +from sqlbuild.sql_values.types import StateSqlValueType BUILD_IDENTITY: str STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] @@ -274,22 +310,23 @@ class LintPreparationRequest(TypedDict): dialect: str def analyze_column_references_json(request_json: str) -> str: ... -def catalogue_json() -> str: ... +def builtin_rules(*, check: RuleCheck) -> list[Rule]: ... +def selected_sql_rule_codes(*, select: list[str], ignore: list[str]) -> list[str]: ... class NativeRulesRequest: """A natively built rules request held until it is evaluated once.""" + def sql_facts_json(self) -> str: ... + def build_rules_request( project: dict[str, object], models: tuple[NativeCompiledProject, list[str]], - sql_tests: list[tuple[object, ...]], - scenarios: list[tuple[object, ...]], + sql_tests: list[CompiledSqlTest], + scenarios: list[CompiledSqlScenario], ) -> NativeRulesRequest: ... def evaluate_rules_request( request: NativeRulesRequest, -) -> tuple[ - list[tuple[bool, str, str, int, int, str, str]], list[str], int, int, int, int, bool -]: ... +) -> tuple[list[Finding], list[str], int, int, int, int, bool]: ... def emit_orjson_report(report: object) -> str: ... def emit_json_report(report: object) -> str: ... @@ -1359,6 +1396,70 @@ def resolve_target_connection_config( def validate_preview_target( *, discovered_inputs: DiscoveredProjectInputs, as_target: str | None, command_name: str ) -> None: ... +def normalize_dbt_profile_connection( + *, resolved: ResolvedDbtProfileOutput +) -> NormalizedDbtProfileConnection: ... +def build_dbt_init_project_toml( + *, + project_name: str, + adapter: str, + target_name: str, + profile_name: str, + dbt_project_dir: str, + profiles_dir: str, + target_path: str, + target_schema: str | None, + target_database: str | None, +) -> str: ... +def load_dbt_manifest_index(*, manifest_path: Path) -> DbtManifestIndex: ... +def build_compile_dbt_manifest_index( + *, manifest_contents: str | None +) -> DbtManifestIndex | None: ... +def build_dbt_manifest_index(*, raw_data: object) -> DbtManifestIndex: ... +def resolve_dbt_manifest_model( + *, manifest: DbtManifestIndex, name: str, package_name: str | None = None +) -> DbtManifestModel: ... +def validate_compile_dbt_model_names( + *, known_model_names: Iterable[str], dbt_manifest: DbtManifestIndex | None +) -> None: ... +def validate_compile_dbt_model_reference( + *, + reference: CompileSqlReference, + model_relative_path: Path, + dbt_manifest: DbtManifestIndex | None, +) -> None: ... +def resolve_sqlbuild_model_dbt_refs( + *, + project: CompiledProjectView, + manifest: DbtManifestIndex, + selected_model_names: Iterable[str] | None = None, +) -> tuple[tuple[CompiledModelView, DbtManifestModel], ...]: ... +def resolve_dbt_reference_relation( + *, + manifest: DbtManifestIndex | None, + ref_kind: str, + ref_name: str, + ref_package: str | None, +) -> str | None: ... +def build_dbt_interop_plan( + *, + command: DbtInteropCommand | str, + dbt_command_argv: Sequence[str], + dbt_ls_nodes: Sequence[DbtLsNode], + sqlbuild_command_argvs: Sequence[Sequence[str]], + selection: DbtInteropSelectionResult, + dbt_required_selector_terms: Sequence[str] = (), + supplemental_dbt_command_argvs: Sequence[Sequence[str]] = (), + warnings: Sequence[str] = (), + sqlbuild_plan_output: PlanOutput | None = None, +) -> DbtInteropPlan: ... +def render_dbt_interop_plan( + *, + plan: DbtInteropPlan, + json_output: bool, + use_color: bool, + display_options: DisplayOptions | None = None, +) -> str: ... def run_debug_command( *, project_dir: Path | None = None, @@ -1425,6 +1526,33 @@ class NativeProgressProjector: ) -> float | None: ... def close(self) -> None: ... +def render_diff_output( + *, + result: DiffExecutionResult, + from_label: str, + to_label: str, + mode_label: str, + use_color: bool, + verbose: bool, + max_column_examples: int, + max_row_only_examples: int, + max_value_length: int | None = ..., + suppress_example_values: bool = ..., + outcome: str | None = ..., +) -> str: ... +def render_diff_error_json( + *, + status: str, + code: str, + message: str, + help_text: str | None = ..., + size_guard: dict[str, object] | None = ..., +) -> str: ... +def run_diff_command(*, request: DiffCommandRequest) -> int: ... +def execute_query_diff( + *, request: DiffCommandRequest, preparation: QueryDiffPreparation +) -> DiffExecutionResult: ... +def machine_output_mode(args: CliNamespace, /) -> bool: ... def current_native_progress_projector() -> NativeProgressProjector | None: ... class ConnectionProgressReporter: @@ -1612,9 +1740,7 @@ class PreparedCompileArtifacts: ) -> WrittenTarget | None: ... def planning_diagnostics(self) -> tuple[CompilerDiagnostic, ...] | None: ... -def finalize_rule_findings_rows( - request: dict[str, object], -) -> list[tuple[bool, str, str, int, int, str, str]]: ... +def finalize_rule_findings_rows(request: dict[str, object]) -> list[Finding]: ... def run_custom_host_json(spec_json: str) -> str: ... def format_sql_json(request_json: str) -> str: ... def format_sql_batch_json(request_json: str) -> str: ... @@ -1628,7 +1754,7 @@ def prepare_lint_sql_batch( requests: list[LintPreparationRequest], ) -> list[tuple[bool, tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None]]: ... def lint_backtick_identifiers(dialect: str) -> bool: ... -def load_config_json(project_dir: str) -> str: ... +def load_rules_config(*, project_dir: Path) -> RulesConfig: ... def selected_codes_json(request_json: str) -> str: ... def render_owned_skill(content: str, input_fingerprint: str) -> str: ... def skill_freshness(content: str | None, input_fingerprint: str) -> str: ... @@ -2737,6 +2863,66 @@ def run_lineage_command(*, request: LineageCommandRequest) -> int: ... def run_refactor_command(*, request: RefactorCommandRequest) -> int: ... def run_janitor_command(*, request: JanitorCommandRequest) -> int: ... def run_freshness_command(*, request: FreshnessCommandRequest) -> int: ... +def run_clone_command(*, request: CloneCommandRequest) -> int: ... +def run_load_command(*, request: LoadCommandRequest) -> int: ... +def run_check_command(*, request: CheckCommandRequest) -> int: ... +def write_check_results( + *, + stream: TextIO, + results: tuple[PythonCheckExecutionResult, ...], + use_color: bool, + check_functions: tuple[DiscoveredCheckFunction, ...] = (), + python_graph: PythonNodeGraph | None = None, +) -> None: ... +def format_check_json(*, results: tuple[PythonCheckExecutionResult, ...]) -> str: ... +def run_test_command(*, request: TestCommandRequest) -> int: ... +def run_seed_command(*, request: SeedCommandRequest) -> int: ... +def format_parameterized_test_label( + *, + name: str, + source_path: Path | None, + parameter_schema: tuple[SqlTestParameterDeclaration, ...], + parameter_values: tuple[tuple[str, SqlValue], ...], +) -> str: ... +def resolve_test_name_width(test_entries: tuple[SqlTestPlanEntry, ...]) -> int: ... +def build_test_expectation_rows( + result: SqlTestExecutionResult, +) -> tuple[NestedProgressChildRow, ...]: ... +def select_load_entries( + *, + discovered_inputs: DiscoveredProjectInputs, + select: tuple[str, ...], + exclude: tuple[str, ...], + target_config: TargetConfig | None, + loader_default_database: str | None = None, + loader_default_schema: str | None = None, +) -> tuple[SourceEntry, ...]: ... +def select_load_reference_entries( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_sources: tuple[SourceEntry, ...], + target_config: TargetConfig | None, + loader_default_database: str | None = None, + loader_default_schema: str | None = None, +) -> tuple[SourceEntry, ...]: ... +def resolve_loader_defaults( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None = None, + target_config: TargetConfig | None = None, + cli_vars: dict[str, object] | None = None, +) -> tuple[str | None, str | None]: ... + +class LoadProgressReporter: + def __init__( + self, *, stream: TextIO, use_color: bool, source_order: dict[str, int], total_count: int + ) -> None: ... + def on_start(self, source: SourceEntry) -> None: ... + def on_progress(self, source: SourceEntry, message: str) -> None: ... + def on_complete(self, result: LoadExecutionResult) -> None: ... + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + def observe_source_freshness_json( *, adapter: object, @@ -2904,6 +3090,69 @@ def _oracle_yaml_load(text: str) -> str: ... def _oracle_toml_load(text: str) -> str: ... # Discovery: header values and authored locations. +def deterministic_migration_event_id( + *, + run_id: str, + target_name: str | None, + origin: MigrationRelation, + destination: MigrationRelation, +) -> str: ... +def deterministic_column_migration_event_id( + *, + run_id: str, + target_name: str | None, + relation: MigrationRelation, + origin_column: str, + destination_column: str, +) -> str: ... +def deterministic_old_name_view_event_id( + *, event_type: OldNameViewEventType, migration_event_id: str +) -> str: ... +def render_event_exists_sql(*, table: str, event_id: str) -> str: ... +def render_event_insert_sql( + *, + table: str, + columns: tuple[str, ...], + column_types: Mapping[str, StateSqlValueType], + values: tuple[object | None, ...], +) -> str: ... +def source_freshness_data_version_hash( + *, + source_name: str, + strategy: SourceFreshnessStrategy | str, + value_kind: SourceFreshnessValueKind | str, + data_version: str, +) -> str: ... +def normalize_source_freshness_data_version( + *, value: object, value_kind: SourceFreshnessValueKind +) -> str: ... +def source_freshness_records_equivalent( + *, + previous_record: SourceFreshnessComparableRecord, + current_record: SourceFreshnessComparableRecord, + lag_tolerance: str | None = None, +) -> bool: ... +def evaluate_source_freshness_age_policy( + *, + policy: SourceFreshnessAgePolicy | None, + data_version: object, + observed_at: datetime, +) -> SourceFreshnessAgeStatus | None: ... +def redact_diagnostic_text(value: str) -> str: ... +def redact_diagnostic_value(*, name: str, value: object) -> object: ... +def sql_rule_lint_key(*, config: LintConfig) -> str: ... +def sql_model_rule_keys( + *, lint_key: str, codes: list[str], models: list[tuple[str, str, str, str | None]] +) -> dict[str, str]: ... +def sql_file_rule_key( + *, lint_key: str, codes: list[str], relative_path: str, contents: str, expanded: bool +) -> str | None: ... +def load_sql_rule_cache(*, path: Path) -> dict[str, tuple[str, str]]: ... +def lookup_sql_rule_cache( + *, entries: dict[str, tuple[str, str]], keys: list[tuple[str, str]] +) -> dict[str, list[Finding]]: ... +def encode_sql_rule_findings(*, findings: list[Finding]) -> str: ... +def write_sql_rule_cache(*, path: Path, entries: dict[str, tuple[str, str]]) -> None: ... def cursor_parse(*, raw: object, cursor_type: str) -> CursorScalar: ... def cursor_try_parse(*, raw: object, cursor_type: str) -> CursorScalar | None: ... def cursor_render(*, value: CursorScalar) -> str: ... @@ -2952,6 +3201,53 @@ def custom_rule_import_closure(*, rule: Rule, project_dir: Path) -> tuple[Path, def custom_rule_implementation_fingerprint( *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] | None = None ) -> str: ... +def custom_rule_identity( + *, + code_identity: str, + runtime_version: str, + code: str, + family: str, + slug: str, + message: str, + remediation: str, + subject: str | None, + implementation_fingerprint: str, + config: RulesConfig, + dialect: str, + project_dir: str, +) -> str: ... + +class CustomRuleCache: + """One custom-rule run's subjects, reused and new results, and its cache file.""" + + def __init__( + self, + *, + path: Path | None, + rules: list[tuple[str, str | None, str | None]], + target_name: str | None, + models: list[tuple[str, str, str | None]], + ) -> None: ... + def pending_readsets(self) -> list[tuple[tuple[str, ...], ...]]: ... + def plan(self, *, digests: list[str | None]) -> dict[str, list[str] | None]: ... + def record( + self, + *, + evaluated: list[ + tuple[ + str, + str, + tuple[tuple[tuple[str, ...], ...], str | None] | None, + list[dict[str, object]], + ] + ], + ) -> None: ... + def findings(self) -> list[Finding]: ... + @property + def hits(self) -> int: ... + @property + def misses(self) -> int: ... + def custom_rule_test_evidence( *, rule: Rule, project_dir: Path ) -> list[tuple[str, int, int, str]]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py b/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py index acb0929b8d..2e302edb70 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py @@ -4,13 +4,12 @@ from typing import Any -from sqlbuild._native import build_project_relation_targets +from sqlbuild._native import build_project_relation_targets, write_check_results from sqlbuild.cli.commands._helpers.check.core import ( build_check_relation_targets, load_results_by_loader_name, record_python_run_state_results, relevant_check_functions, - write_check_results, ) from sqlbuild.cli.commands.models import ( BuildCommandRequest, diff --git a/src/sqlbuild/cli/commands/_helpers/check/core.py b/src/sqlbuild/cli/commands/_helpers/check/core.py index 6e84a0da06..3ff34e394d 100644 --- a/src/sqlbuild/cli/commands/_helpers/check/core.py +++ b/src/sqlbuild/cli/commands/_helpers/check/core.py @@ -2,17 +2,11 @@ from __future__ import annotations -import json from collections.abc import Callable -from typing import Any, TextIO +from typing import Any from sqlbuild._native import ( - NativeProgressProjector, - TerminalEventIndex, build_python_relation_targets, - current_native_progress_projector, - current_terminal_event_index, - transitive_closure_many, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup @@ -25,11 +19,8 @@ from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector from sqlbuild.compiler.python_nodes.main.selectors import resolve_python_nodes_from_selectors from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonNode, PythonNodeGraph, PythonSqlRunLifecyclePlan, ) @@ -43,7 +34,6 @@ PythonNodeRunState, PythonNodeRuntime, ) -from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.provider.main.runtime import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind @@ -52,74 +42,6 @@ _IGNORED_EXCLUDE_SELECTOR_ERROR_CODES: frozenset[str] = frozenset({"S007", "S008", "S009"}) -def resolve_selected_check_names( - *, graph: PythonNodeGraph, select: tuple[str, ...], exclude: tuple[str, ...] -) -> frozenset[str]: - """Resolve sqb check selectors into check names only.""" - - _validate_check_selectors(select=(*select, *exclude)) - if select: - selected_names: frozenset[str] = resolve_python_nodes_from_selectors( - select=select, exclude=exclude, graph=graph - ) - return _check_names_from_selection(graph=graph, selected_names=selected_names) - check_names: frozenset[str] = frozenset( - node.name for node in graph.nodes if node.kind == PythonNodeKind.CHECK - ) - excluded: frozenset[str] = ( - resolve_python_nodes_from_selectors(select=exclude, exclude=(), graph=graph) - if exclude - else frozenset() - ) - return frozenset(check_names - excluded) - - -def check_dependency_closure( - *, graph: PythonNodeGraph, check_names: frozenset[str] -) -> frozenset[str]: - """Return transitive executable Python dependencies of selected checks.""" - - dependencies: frozenset[str] = transitive_closure_many( - starts=check_names, edges=graph.upstream_deps, include_starts=False - ) - return dependencies - check_names - - -def _validate_check_selectors(*, select: tuple[str, ...]) -> None: - raw_selector: str - for raw_selector in select: - token: str - for token in raw_selector.split(): - part: str - for part in token.split(","): - parsed: ParsedSelector | PathSelector = parse_project_selector(part) - if isinstance(parsed, PathSelector) or parsed.upstream or parsed.downstream: - raise CliUserError( - "sqb check selectors do not support graph operators; " - "select checks directly without '+', '~', or path-between expansion", - code="C681", - ) - - -def _check_names_from_selection( - *, graph: PythonNodeGraph, selected_names: frozenset[str] -) -> frozenset[str]: - non_check_names: tuple[str, ...] = tuple( - sorted( - name - for name in selected_names - if graph.nodes_by_name[name].kind != PythonNodeKind.CHECK - ) - ) - if non_check_names: - raise CliUserError( - "sqb check selectors must resolve only to Python checks; non-check selections: " - + ", ".join(non_check_names), - code="C682", - ) - return selected_names - - def record_python_run_state_results( *, discovered_inputs: DiscoveredProjectInputs, @@ -265,99 +187,6 @@ def load_results_by_loader_name( } -def write_check_results( - *, - stream: TextIO, - results: tuple[PythonCheckExecutionResult, ...], - use_color: bool, - check_functions: tuple[DiscoveredCheckFunction, ...] = (), - python_graph: PythonNodeGraph | None = None, -) -> None: - """Write human-readable check result rows.""" - - style: CliStyle = CliStyle(use_color=use_color) - projector: NativeProgressProjector | None = current_native_progress_projector() - result_by_name: dict[str, PythonCheckExecutionResult] = { - result.node_name: result for result in results - } - grouped_names: dict[str, list[str]] = _group_check_names( - results=results, - check_functions=check_functions, - python_graph=python_graph, - ) - stream.write("\n") - group_label: str - result_names: list[str] - for group_label, result_names in grouped_names.items(): - stream.write(f"Python checks: {group_label}\n\n") - for result_name in result_names: - result: PythonCheckExecutionResult = result_by_name[result_name] - duration_ms: float | None = ( - projector.resource_terminal_duration( - resource_name=result.node_name, - resource_id=f"check:{result.node_name}", - ) - if projector is not None - else None - ) - _write_check_result_row( - stream=stream, - result=result, - style=style, - duration_ms=duration_ms, - ) - if projector is not None: - _ = projector.consume_resource_terminal( - resource_name=result.node_name, - resource_id=f"check:{result.node_name}", - ) - stream.write("\n") - - -def format_check_json(*, results: tuple[PythonCheckExecutionResult, ...]) -> str: - """Format check execution JSON.""" - - projector: TerminalEventIndex | None = current_terminal_event_index() - if projector is not None: - results = tuple( - result - for result in results - if projector.resource_terminal( - resource_name=result.node_name, - resource_id=f"check:{result.node_name}", - ) - is not None - ) - payload: dict[str, object] = { - "version": 1, - "command": "check", - "status": "failed" if check_results_failed(results) else "success", - "summary": { - "pass_count": sum(1 for result in results if result.passed), - "warn_count": sum(1 for result in results if result.warned), - "fail_count": sum(1 for result in results if result.failed), - "total_count": len(results), - }, - "checks": [ - { - "kind": "python_check", - "name": result.node_name, - "display_name": result.node_name, - "check_id": f"python_check:{result.node_name}", - "status": "pass" if result.passed else "warn" if result.warned else "fail", - "passed": result.passed, - "severity": result.severity.value, - "message": result.message, - "error_message": result.error_message, - "metadata": result.metadata, - "warnings": list(result.warning_messages) or None, - } - for result in results - ], - } - return json.dumps(payload, indent=2) + "\n" - - def check_results_failed(results: tuple[PythonCheckExecutionResult, ...]) -> bool: """Return whether any Python check produced an error-severity failure.""" @@ -528,67 +357,3 @@ def _check_sql_ref_location( def _sql_ref_sort_key(ref: SqlResourceRef) -> tuple[str, str]: return (ref.kind.value, ref.name) - - -def _group_check_names( - *, - results: tuple[PythonCheckExecutionResult, ...], - check_functions: tuple[DiscoveredCheckFunction, ...], - python_graph: PythonNodeGraph | None, -) -> dict[str, list[str]]: - check_by_name: dict[str, DiscoveredCheckFunction] = { - check.name: check for check in check_functions - } - groups: dict[str, list[str]] = {} - result: PythonCheckExecutionResult - for result in results: - group_label: str = _check_group_label( - check=check_by_name.get(result.node_name), - result_name=result.node_name, - python_graph=python_graph, - ) - groups.setdefault(group_label, []).append(result.node_name) - return groups - - -def _check_group_label( - *, - check: DiscoveredCheckFunction | None, - result_name: str, - python_graph: PythonNodeGraph | None, -) -> str: - if python_graph is not None: - node: DiscoveredPythonNode | None = python_graph.nodes_by_name.get(result_name) - upstream_names: tuple[str, ...] = ( - *python_graph.upstream_deps.get(result_name, ()), - *(() if node is None else (sql_ref.name for sql_ref in node.sql_deps)), - ) - if len(upstream_names) == 1: - return upstream_names[0] - if len(upstream_names) > 1: - return ( - check.group if check is not None and check.group is not None else "multi-dependency" - ) - if check is not None and check.group is not None: - return check.group - return "ungrouped" - - -def _write_check_result_row( - *, - stream: TextIO, - result: PythonCheckExecutionResult, - style: CliStyle, - duration_ms: float | None, -) -> None: - status: str = "PASS" if result.passed else "WARN" if result.warned else "FAIL" - stream.write(f" {'check':<10}{result.node_name:<50} {style.status(status=status)}") - if duration_ms is not None: - stream.write(f" {duration_ms / 1000.0:.2f}s") - detail: str | None = result.error_message or result.message - if detail: - stream.write(f" {detail}") - stream.write("\n") - warning_message: str - for warning_message in result.warning_messages: - stream.write(f" {style.warning(warning_message)}\n") diff --git a/src/sqlbuild/cli/commands/_helpers/check/execution.py b/src/sqlbuild/cli/commands/_helpers/check/execution.py deleted file mode 100644 index 973e138dbd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/check/execution.py +++ /dev/null @@ -1,272 +0,0 @@ -"""Check command execution phases.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild._native import ( - ExecutionEventWriter, - NativeProgressProjector, - build_project_relation_targets, - current_native_progress_projector, -) -from sqlbuild.cli.commands._helpers.check.core import ( - build_check_relation_targets, - load_results_by_loader_name, - record_python_run_state_results, - resolve_selected_check_names, - run_check_read_side_dependencies, -) -from sqlbuild.cli.commands.models import ( - CheckCommandRequest, - CheckExecutionPreparation, - CheckInvocation, -) -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.main.run_lifecycle import build_python_sql_run_lifecycle -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) -from sqlbuild.executor.python_nodes.main.checks import execute_python_checks -from sqlbuild.executor.python_nodes.main.ingress import execute_ingress_python_loader_nodes -from sqlbuild.executor.python_nodes.models import ( - IngressCallbacks, - PythonCheckCallbacks, - PythonCheckExecutionResult, - PythonIngressLoaderExecutorResult, - PythonNodeExecutionResult, - PythonNodeRuntime, -) -from sqlbuild.provider.classes.container import ProviderContainer -from sqlbuild.python_nodes.models import SqlResourceRef - - -def prepare_check_execution( - *, - request: CheckCommandRequest, - invocation: CheckInvocation, - pipeline_result: CompilePipelineResult, -) -> CheckExecutionPreparation: - """Prepare Python graph selection and relation context for check execution.""" - - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=invocation.discovered_inputs - ) - reject_unit_test_selectors( - select=request.select, - exclude=request.exclude, - discovered_inputs=invocation.discovered_inputs, - ) - check_names: frozenset[str] = resolve_selected_check_names( - graph=python_graph, - select=request.select, - exclude=request.exclude, - ) - check_functions: tuple[DiscoveredCheckFunction, ...] = tuple( - check for check in invocation.discovered_inputs.check_functions if check.name in check_names - ) - lifecycle_plan: PythonSqlRunLifecyclePlan = build_python_sql_run_lifecycle( - selection=PythonSqlRunSelection( - sql_keys=frozenset(), - python_node_names=frozenset(), - ), - python_graph=python_graph, - ) - relation_targets: dict[SqlResourceRef, str] = build_check_relation_targets( - adapter=invocation.adapter, - pipeline_result=pipeline_result, - python_graph=python_graph, - selected_python_names=check_names, - ) - relation_refs: frozenset[SqlResourceRef] = python_graph.selected_sql_refs( - selected_names=check_names - ) - return CheckExecutionPreparation( - python_graph=python_graph, - check_functions=check_functions, - lifecycle_plan=lifecycle_plan, - relation_targets=relation_targets, - relation_refs=relation_refs, - default_database=_default_database(invocation=invocation, pipeline_result=pipeline_result), - default_schema=_default_schema(invocation=invocation, pipeline_result=pipeline_result), - project_relations=build_project_relation_targets( - adapter=invocation.adapter, plan_output=pipeline_result.plan_output - ), - ) - - -def execute_check_plan( - *, - invocation: CheckInvocation, - pipeline_result: CompilePipelineResult, - preparation: CheckExecutionPreparation, - providers: ProviderContainer, -) -> tuple[PythonCheckExecutionResult, ...]: - """Execute selected Python checks with their Python dependencies.""" - - connection: Any = invocation.adapter.connect(invocation.connection_config) - event_writer: ExecutionEventWriter = ExecutionEventWriter() - projector: NativeProgressProjector | None = current_native_progress_projector() - if projector is not None: - projector.configure_resources( - ordinals={ - check.name: index - for index, check in enumerate(preparation.check_functions, start=1) - }, - total=len(preparation.check_functions), - ) - on_check_complete: Callable[[PythonCheckExecutionResult], None] = _check_completion_adapter( - event_writer=event_writer, - pipeline_result=pipeline_result, - ) - try: - ingress_result: PythonIngressLoaderExecutorResult = _execute_check_ingress( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - connection=connection, - providers=providers, - ) - read_side_results: tuple[PythonNodeExecutionResult, ...] = _execute_check_read_side( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - connection=connection, - ingress_result=ingress_result, - providers=providers, - ) - return execute_python_checks( - check_functions=preparation.check_functions, - python_graph=preparation.python_graph, - upstream_python_results=(*ingress_result.python_results, *read_side_results), - upstream_load_results=ingress_result.load_results, - upstream_load_results_by_loader_name=load_results_by_loader_name( - source_map=pipeline_result.plan_output.source_map, - load_results=ingress_result.load_results, - ), - runtime=PythonNodeRuntime( - adapter=invocation.adapter, - connection_config=invocation.connection_config, - connection=connection, - run_id=pipeline_result.project.run_id, - target=pipeline_result.project.effective_target_name, - vars=pipeline_result.project.effective_vars, - is_reload=False, - default_database=preparation.default_database, - default_schema=preparation.default_schema, - relation_targets=preparation.relation_targets, - project_relations=preparation.project_relations, - providers=providers, - ), - run_state=ingress_result.run_state, - require_upstream_results=False, - callbacks=PythonCheckCallbacks(on_check_complete=on_check_complete), - ) - finally: - event_writer.close() - invocation.adapter.close(connection) - - -def _check_completion_adapter( - *, - event_writer: ExecutionEventWriter, - pipeline_result: CompilePipelineResult, -) -> Callable[[PythonCheckExecutionResult], None]: - def _on_complete(result: PythonCheckExecutionResult) -> None: - event_writer.write_build_result( - result=result, - plan=pipeline_result.plan_output, - command="check", - ) - - return _on_complete - - -def _execute_check_ingress( - *, - invocation: CheckInvocation, - pipeline_result: CompilePipelineResult, - preparation: CheckExecutionPreparation, - connection: Any, - providers: ProviderContainer, -) -> PythonIngressLoaderExecutorResult: - ingress_result: PythonIngressLoaderExecutorResult = execute_ingress_python_loader_nodes( - python_graph=preparation.python_graph, - selected_python_names=preparation.lifecycle_plan.ingress_python_node_names, - loader_functions=invocation.discovered_inputs.loader_functions, - source_map=pipeline_result.plan_output.source_map, - runtime=PythonNodeRuntime( - adapter=invocation.adapter, - connection_config=invocation.connection_config, - connection=connection, - run_id=pipeline_result.project.run_id, - target=pipeline_result.project.effective_target_name, - vars=pipeline_result.project.effective_vars, - is_reload=False, - default_database=preparation.default_database, - default_schema=preparation.default_schema, - relation_targets=preparation.relation_targets, - providers=providers, - ), - callbacks=IngressCallbacks(use_color=invocation.use_color), - ) - record_python_run_state_results( - discovered_inputs=invocation.discovered_inputs, - run_state=ingress_result.run_state, - python_results=ingress_result.python_results, - load_results=ingress_result.load_results, - source_map=pipeline_result.plan_output.source_map, - ) - return ingress_result - - -def _execute_check_read_side( - *, - invocation: CheckInvocation, - pipeline_result: CompilePipelineResult, - preparation: CheckExecutionPreparation, - connection: Any, - ingress_result: PythonIngressLoaderExecutorResult, - providers: ProviderContainer, -) -> tuple[PythonNodeExecutionResult, ...]: - read_side_results: tuple[PythonNodeExecutionResult, ...] = run_check_read_side_dependencies( - adapter=invocation.adapter, - connection_config=invocation.connection_config, - connection=connection, - pipeline_result=pipeline_result, - python_graph=preparation.python_graph, - lifecycle_plan=preparation.lifecycle_plan, - relation_targets=preparation.relation_targets, - validation_refs=preparation.relation_refs, - project_relations=preparation.project_relations, - providers=providers, - ) - record_python_run_state_results( - discovered_inputs=invocation.discovered_inputs, - run_state=ingress_result.run_state, - python_results=read_side_results, - source_map=pipeline_result.plan_output.source_map, - ) - return read_side_results - - -def _default_database( - *, invocation: CheckInvocation, pipeline_result: CompilePipelineResult -) -> str | None: - if pipeline_result.project.effective_target_database is not None: - return pipeline_result.project.effective_target_database - return invocation.adapter.default_database() - - -def _default_schema( - *, invocation: CheckInvocation, pipeline_result: CompilePipelineResult -) -> str | None: - if pipeline_result.project.effective_target_schema is not None: - return pipeline_result.project.effective_target_schema - return invocation.adapter.default_schema() diff --git a/src/sqlbuild/cli/commands/_helpers/check/invocation.py b/src/sqlbuild/cli/commands/_helpers/check/invocation.py deleted file mode 100644 index e0d0b8556d..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/check/invocation.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Check command invocation resolution phase.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( - resolve_reported_command_invocation, -) -from sqlbuild.cli.commands.models import CheckCommandRequest, CheckInvocation - - -def resolve_check_invocation(*, request: CheckCommandRequest) -> CheckInvocation: - """Resolve discovery, adapter, connection, and reporters for check.""" - - return resolve_reported_command_invocation(request=request, invocation_type=CheckInvocation) diff --git a/src/sqlbuild/cli/commands/_helpers/check/outputs.py b/src/sqlbuild/cli/commands/_helpers/check/outputs.py deleted file mode 100644 index ffc4034ea7..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/check/outputs.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Check command output writing and exit-code phases.""" - -from __future__ import annotations - -from sqlbuild._native import format_summary_footer, write_execution_json_output -from sqlbuild.cli.commands._helpers.check.core import format_check_json, write_check_results -from sqlbuild.cli.commands.models import ( - CheckCommandRequest, - CheckExecutionPreparation, - CheckInvocation, -) -from sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target import ( - write_python_check_runtime_target, -) -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult - - -def write_check_completion_output( - *, - request: CheckCommandRequest, - invocation: CheckInvocation, - preparation: CheckExecutionPreparation, - results: tuple[PythonCheckExecutionResult, ...], -) -> None: - """Write check rows, summary, runtime artifact, and optional JSON output.""" - - write_check_results( - stream=invocation.progress_stream, - results=results, - use_color=invocation.use_color, - check_functions=preparation.check_functions, - python_graph=preparation.python_graph, - ) - invocation.progress_stream.write( - "\n" - + format_summary_footer( - counts=( - ("PASS", _pass_count(results)), - ("WARN", _warn_count(results=results)), - ("FAIL", _fail_count(results)), - ("TOTAL", len(results)), - ), - use_color=invocation.use_color, - ) - + "\n" - ) - invocation.progress_stream.flush() - write_python_check_runtime_target( - target_dir=invocation.effective_project_dir / "target", results=results - ) - write_execution_json_output( - payload=format_check_json(results=results), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def resolve_check_exit_code(results: tuple[PythonCheckExecutionResult, ...]) -> int: - """Resolve the check exit code from failed Python check results.""" - - return 0 if _fail_count(results) == 0 else 1 - - -def _pass_count(results: tuple[PythonCheckExecutionResult, ...]) -> int: - return sum(1 for result in results if result.passed) - - -def _warn_count(results: tuple[PythonCheckExecutionResult, ...]) -> int: - return sum(1 for result in results if result.warned) - - -def _fail_count(results: tuple[PythonCheckExecutionResult, ...]) -> int: - return sum(1 for result in results if result.failed) diff --git a/src/sqlbuild/cli/commands/_helpers/check/planning.py b/src/sqlbuild/cli/commands/_helpers/check/planning.py deleted file mode 100644 index 629d3d9987..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/check/planning.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Check command plan compilation phase.""" - -from __future__ import annotations - -from sqlbuild._native import build_static_pipeline_result, sql_ref_key -from sqlbuild.cli.commands._helpers.check.core import ( - check_dependency_closure, - resolve_selected_check_names, -) -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.models import CheckCommandRequest, CheckInvocation -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, - StaticCommandContext, -) -from sqlbuild.compiler.planner.main.commands.relation_plan import build_relation_command_plan -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.python_nodes.models import SqlResourceRef - - -def compile_check_plan( - *, request: CheckCommandRequest, invocation: CheckInvocation -) -> CompilePipelineResult: - """Compile the project for Python check execution.""" - - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=invocation.discovered_inputs - ) - reject_unit_test_selectors( - select=request.select, - exclude=request.exclude, - discovered_inputs=invocation.discovered_inputs, - ) - check_names: frozenset[str] = resolve_selected_check_names( - graph=python_graph, - select=request.select, - exclude=request.exclude, - ) - required_refs: frozenset[SqlResourceRef] = python_graph.selected_sql_refs( - selected_names=check_names - ) - dependency_names: frozenset[str] = check_dependency_closure( - graph=python_graph, check_names=check_names - ) - required_loader_names: frozenset[str] = frozenset( - name - for name in dependency_names - if python_graph.nodes_by_name[name].kind == PythonNodeKind.LOADER - ) - required_source_names: frozenset[str] = _required_loader_source_names( - invocation=invocation, - required_loader_names=required_loader_names, - ) - selected_keys: frozenset[CompiledObjectKey] = frozenset( - sql_ref_key(ref) for ref in required_refs - ) | frozenset( - CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=name) - for name in required_source_names - ) - context: StaticCommandContext = compile_static_command_context( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - options=CompilePipelineOptions( - selected_target=request.selected_target, - no_sql_validation=request.no_sql_validation, - connection_config=invocation.connection_config, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ), - selected_keys=selected_keys, - relation_keys=selected_keys, - on_progress=invocation.planning_progress.on_progress, - ) - return build_static_pipeline_result( - context=context, - plan_output=build_relation_command_plan( - project=context.project, - scope=context.scope, - relations=context.relations, - ), - ) - - -def _required_loader_source_names( - *, invocation: CheckInvocation, required_loader_names: frozenset[str] -) -> frozenset[str]: - names: set[str] = set() - for source_file in invocation.discovered_inputs.source_files: - for source in source_file.source_entries: - if source.loader in required_loader_names: - names.add(source.name) - return frozenset(names) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/__init__.py b/src/sqlbuild/cli/commands/_helpers/clone/__init__.py deleted file mode 100644 index ea2285a6d1..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Clone command helper package.""" diff --git a/src/sqlbuild/cli/commands/_helpers/clone/connections.py b/src/sqlbuild/cli/commands/_helpers/clone/connections.py deleted file mode 100644 index 0f6a691b29..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/connections.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Clone command connection phases.""" - -from __future__ import annotations - -import time -from typing import Any - -from sqlbuild._native import ( - PlanningProgressReporter, - resolve_target_connection_config, -) -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneConnectionContext, - CloneInvocation, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.phase_line import format_phase_line -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def connect_clone_targets( - *, request: CloneCommandRequest, invocation: CloneInvocation -) -> CloneConnectionContext: - """Resolve and connect the destination target used for the whole clone.""" - - destination_connection_config: dict[str, object] = resolve_target_connection_config( - discovered_inputs=invocation.discovered_inputs, - project_dir=invocation.effective_project_dir, - target_name=invocation.destination_target_name, - cli_vars=request.cli_vars, - ) - progress: PlanningProgressReporter = PlanningProgressReporter( - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - progress.start(f"Connecting to {invocation.adapter_name}...") - connect_start: float = time.monotonic() - with OperationLifecycle(operation_kind="clone", operation_name="clone_target_connection"): - destination_connection: Any = invocation.adapter.connect(destination_connection_config) - progress.complete_styled( - format_phase_line( - style=CliStyle(use_color=invocation.use_color), - ok=True, - label="Warehouse connected", - summary=(f"{invocation.adapter_name} ({time.monotonic() - connect_start:.2f}s)"), - ) - ) - return CloneConnectionContext( - destination_connection_config=destination_connection_config, - destination_connection=destination_connection, - ) - - -def close_clone_targets( - *, invocation: CloneInvocation, connection_context: CloneConnectionContext -) -> None: - """Close the clone destination connection.""" - - invocation.adapter.close(connection_context.destination_connection) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/execution.py b/src/sqlbuild/cli/commands/_helpers/clone/execution.py deleted file mode 100644 index b8316bb9ce..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/execution.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Clone command execution phases.""" - -from __future__ import annotations - -import time -from typing import TextIO - -from sqlbuild._native import ExecutionEventWriter -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.clone.fingerprint_progress import ( - create_clone_fingerprint_progress_reporter, -) -from sqlbuild.cli.commands._helpers.clone.output import ( - clone_relation_flow_text, - render_clone_item_line, -) -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneConnectionContext, - CloneExecutionPreparation, - CloneInvocation, - CloneRunOutcome, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - FunctionPlanEntry, - ModelPlanEntry, - SeedPlanEntry, -) -from sqlbuild.executor.clone.main.build_destination_transient_models import ( - build_destination_transient_models, -) -from sqlbuild.executor.clone.main.build_retention_requests import ( - build_destination_retention_requests, -) -from sqlbuild.executor.clone.main.execute import execute_clone -from sqlbuild.executor.clone.main.fingerprinting import copy_clone_fingerprints -from sqlbuild.executor.clone.models import ( - CloneExecutionInput, - CloneExecutionResult, - CloneItemResult, - CloneSourceEntries, -) -from sqlbuild.executor.clone.types import CloneFingerprintProgressReporter, CloneItemCallback -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import RetentionDecreasePolicy - - -def execute_clone_plan( - *, - request: CloneCommandRequest, - invocation: CloneInvocation, - connection_context: CloneConnectionContext, - preparation: CloneExecutionPreparation, -) -> CloneRunOutcome: - """Execute the direct clone plan and copy fingerprints.""" - - clone_start: float = time.monotonic() - resource_types_by_name: dict[str, str] = _clone_resource_types_by_name(preparation=preparation) - event_writer: ExecutionEventWriter = ExecutionEventWriter(path=request.event_output_path) - relation_width: int = _clone_relation_width(preparation=preparation) - on_item: CloneItemCallback = _build_on_clone_item( - stream=invocation.progress_stream, - use_color=invocation.use_color, - event_writer=event_writer, - resource_types_by_name=resource_types_by_name, - relation_width=relation_width, - ) - try: - result: CloneExecutionResult = execute_clone( - inputs=CloneExecutionInput( - source_entries=CloneSourceEntries( - origin=preparation.pipeline_result.origin_source_entries, - destination=preparation.pipeline_result.destination_source_entries, - ), - origin_model_entries=preparation.pipeline_result.origin_model_entries, - destination_model_entries=preparation.pipeline_result.destination_model_entries, - origin_seed_entries=preparation.pipeline_result.origin_seed_entries, - destination_seed_entries=preparation.pipeline_result.destination_seed_entries, - destination_function_entries=preparation.pipeline_result.destination_function_entries, - execution_order=preparation.pipeline_result.clone_plan.execution_order, - adapter=invocation.adapter, - destination_connection=connection_context.destination_connection, - hard_copy=request.hard_copy, - run_id=preparation.pipeline_result.destination_project.run_id, - query_change_tracking=preparation.pipeline_result.destination_project.settings.query_change_tracking, - upstream_deps=preparation.pipeline_result.clone_plan.upstream_deps, - dependency_locations=_clone_dependency_locations( - preparation=preparation, - adapter=invocation.adapter, - ), - destination_transient_models=build_destination_transient_models( - project=preparation.pipeline_result.destination_project - ), - destination_retention_requests=build_destination_retention_requests( - project=preparation.pipeline_result.destination_project, - adapter_name=invocation.adapter_name, - selected_model_names=frozenset( - entry.name - for entry in preparation.pipeline_result.destination_model_entries - ), - namespace_owned=resolve_target_config( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - target_name=invocation.destination_target_name, - ).owns_time_travel_retention_namespace, - ), - allow_namespace_retention_decrease=( - resolve_target_config( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - target_name=invocation.destination_target_name, - ).time_travel_retention_decrease - == RetentionDecreasePolicy.ALLOW - ), - on_item=on_item, - ) - ) - finally: - event_writer.close() - fingerprint_progress: CloneFingerprintProgressReporter = ( - create_clone_fingerprint_progress_reporter( - stream=invocation.progress_stream, use_color=invocation.use_color - ) - ) - try: - with OperationLifecycle(operation_kind="clone", operation_name="clone_finalization"): - copy_clone_fingerprints( - result=result, - origin_model_entries=preparation.pipeline_result.origin_model_entries, - destination_model_entries=preparation.pipeline_result.destination_model_entries, - origin_seed_entries=preparation.pipeline_result.origin_seed_entries, - destination_seed_entries=preparation.pipeline_result.destination_seed_entries, - adapter=invocation.adapter, - destination_connection=connection_context.destination_connection, - run_id=preparation.pipeline_result.destination_project.run_id, - query_change_tracking=preparation.pipeline_result.destination_project.settings.query_change_tracking, - on_progress=fingerprint_progress, - ) - except BaseException: - fingerprint_progress.write_interrupted() - raise - return CloneRunOutcome(result=result, elapsed=time.monotonic() - clone_start) - - -def _clone_dependency_locations( - *, preparation: CloneExecutionPreparation, adapter: BaseAdapter -) -> dict[CompiledObjectKey, CompiledRelationLocation]: - entries: tuple[ - CloneSourcePlanEntry | ModelPlanEntry | SeedPlanEntry | FunctionPlanEntry, ... - ] = ( - *preparation.pipeline_result.destination_source_entries, - *preparation.pipeline_result.destination_model_entries, - *preparation.pipeline_result.destination_seed_entries, - *preparation.pipeline_result.destination_function_entries, - ) - locations: dict[CompiledObjectKey, CompiledRelationLocation] = { - entry.key: entry.destination for entry in entries - } - source_map: dict[str, SourceEntry] = ( - preparation.pipeline_result.clone_plan.source_read_map - or preparation.pipeline_result.clone_plan.source_map - ) - for name, source in source_map.items(): - if source.table is None: - continue - key: CompiledObjectKey = CompiledObjectKey(CompiledResourceType.SOURCE, name) - locations[key] = CompiledRelationLocation( - database=source.database, - schema=source.schema, - name=source.table, - qualified_name=( - adapter.render_qualified_name( - database=source.database, - schema=source.schema, - name=source.table, - ) - or source.table - ), - ) - return locations - - -def _clone_resource_types_by_name(*, preparation: CloneExecutionPreparation) -> dict[str, str]: - entries: tuple[ - CloneSourcePlanEntry | ModelPlanEntry | SeedPlanEntry | FunctionPlanEntry, ... - ] = ( - *preparation.pipeline_result.destination_source_entries, - *preparation.pipeline_result.destination_model_entries, - *preparation.pipeline_result.destination_seed_entries, - *preparation.pipeline_result.destination_function_entries, - ) - return {entry.name: str(entry.key.resource_type) for entry in entries} - - -def _build_on_clone_item( - *, - stream: TextIO, - use_color: bool, - event_writer: ExecutionEventWriter, - resource_types_by_name: dict[str, str], - relation_width: int, -) -> CloneItemCallback: - def _on_clone_item(*, index: int, total: int, item: CloneItemResult) -> None: - stream.write( - render_clone_item_line( - index=index, - total=total, - item=item, - use_color=use_color, - relation_width=relation_width, - ) - + "\n" - ) - stream.flush() - event_writer.write_clone_result( - item=item, - resource_type=resource_types_by_name[item.name], - ) - - return _on_clone_item - - -def _clone_relation_width(*, preparation: CloneExecutionPreparation) -> int: - origin_entries: tuple[CloneSourcePlanEntry | ModelPlanEntry | SeedPlanEntry, ...] = ( - *preparation.pipeline_result.origin_source_entries, - *preparation.pipeline_result.origin_model_entries, - *preparation.pipeline_result.origin_seed_entries, - ) - origin_relations: dict[str, str | None] = { - entry.name: entry.destination.qualified_name or entry.destination.name - for entry in origin_entries - } - destination_entries: tuple[ - CloneSourcePlanEntry | ModelPlanEntry | SeedPlanEntry | FunctionPlanEntry, ... - ] = ( - *preparation.pipeline_result.destination_source_entries, - *preparation.pipeline_result.destination_model_entries, - *preparation.pipeline_result.destination_seed_entries, - *preparation.pipeline_result.destination_function_entries, - ) - flows: tuple[str, ...] = tuple( - clone_relation_flow_text( - name=entry.name, - origin_relation=origin_relations.get(entry.name), - destination_relation=entry.destination.qualified_name or entry.destination.name, - ) - for entry in destination_entries - ) - return max((len(flow) for flow in flows), default=0) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/fingerprint_progress.py b/src/sqlbuild/cli/commands/_helpers/clone/fingerprint_progress.py deleted file mode 100644 index 39b77edcf6..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/fingerprint_progress.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Clone fingerprint finalization progress reporting.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.cli.commands._helpers.clone.output import ( - render_clone_fingerprint_interrupted_line, - render_clone_fingerprint_progress_line, -) -from sqlbuild.executor.clone.types import CloneFingerprintProgressReporter - - -class _CloneFingerprintProgressReporter: - def __init__(self, *, stream: TextIO, use_color: bool) -> None: - self._stream: TextIO = stream - self._use_color: bool = use_color - self._confirmed: int = 0 - self._total: int = 0 - self._pending_identities: tuple[str, ...] = () - - def __call__(self, *, completed: int, total: int, pending_identities: tuple[str, ...]) -> None: - self._confirmed = completed - self._total = total - self._pending_identities = pending_identities - if completed == 0: - return - self._stream.write( - render_clone_fingerprint_progress_line( - completed=completed, - total=total, - use_color=self._use_color, - ) - + "\n" - ) - self._stream.flush() - - def write_interrupted(self) -> None: - if self._total <= self._confirmed: - return - self._stream.write( - render_clone_fingerprint_interrupted_line( - completed=self._confirmed, - total=self._total, - pending_identities=self._pending_identities, - use_color=self._use_color, - ) - + "\n" - ) - self._stream.flush() - - -def create_clone_fingerprint_progress_reporter( - *, stream: TextIO, use_color: bool -) -> CloneFingerprintProgressReporter: - """Create a stateful reporter for one clone finalization phase.""" - - return _CloneFingerprintProgressReporter(stream=stream, use_color=use_color) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/invocation.py b/src/sqlbuild/cli/commands/_helpers/clone/invocation.py deleted file mode 100644 index c1a07cb339..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/invocation.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Clone command invocation resolution phase.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import PlanningProgressReporter, discover_project_inputs -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.clone.validation import validate_clone_request -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import CloneCommandRequest, CloneInvocation -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name - - -def resolve_clone_invocation(*, request: CloneCommandRequest) -> CloneInvocation: - """Resolve discovery, adapter, and output context for clone.""" - - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - use_color: bool = not request.no_color and supports_color() - progress_stream: TextIO = sys.stdout - progress: PlanningProgressReporter = PlanningProgressReporter( - stream=progress_stream, - use_color=use_color, - ) - progress.start("Discovering project...") - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - progress.complete("Discovered project.") - reject_unit_test_selectors( - select=request.select, exclude=request.exclude, discovered_inputs=discovered_inputs - ) - destination_target_name: str | None = request.destination_target_name - if destination_target_name is None: - destination_target_name = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - if destination_target_name is None: - raise CliUserError( - "clone requires --to when the project has no active target", - code="C406", - ) - validate_clone_request( - discovered_inputs=discovered_inputs, - origin_target_name=request.origin_target_name, - destination_target_name=destination_target_name, - ) - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, project_dir=effective_project_dir - ) - return CloneInvocation( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - adapter_name=adapter_name, - adapter=adapter, - destination_target_name=destination_target_name, - use_color=use_color, - progress_stream=progress_stream, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/output.py b/src/sqlbuild/cli/commands/_helpers/clone/output.py deleted file mode 100644 index 3d0b80a50b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/output.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Clone command output helpers.""" - -from __future__ import annotations - -import re - -from sqlbuild._native import format_completion_line, format_summary_footer -from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult -from sqlbuild.executor.clone.types import CloneAction, CloneStatus -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.types import CompletionState - -_QUOTED_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile( - r'("(?:[^"]|"")*"|`(?:[^`]|``)*`|\[(?:[^\]]|\]\])*\])' -) - - -def render_clone_header( - *, origin_target_name: str, destination_target_name: str, total: int, use_color: bool -) -> str: - """Render the persistent clone header naming origin, destination, and resource count.""" - - style: CliStyle = CliStyle(use_color=use_color) - return ( - f"{style.title('sqb clone')} " - f"{style.label('origin=')}{style.value(origin_target_name)} " - f"{style.label('destination=')}{style.value(destination_target_name)} " - f"{style.muted(f'({total} resource{"" if total == 1 else "s"})')}" - ) - - -def _clone_status_text(status: CloneStatus) -> str: - if status == CloneStatus.WARNING: - return "WARN" - if status == CloneStatus.FAILED: - return "FAIL" - return "OK" - - -def clone_relation_flow_text( - *, name: str, origin_relation: str | None, destination_relation: str | None -) -> str: - """Return the normalized plain-text relation flow used for clone progress.""" - - if origin_relation is None or destination_relation is None: - return name.lower() - return ( - f"{clone_relation_display_name(origin_relation)} -> " - f"{clone_relation_display_name(destination_relation)}" - ) - - -def render_clone_fingerprint_progress_line(*, completed: int, total: int, use_color: bool) -> str: - """Render confirmed clone fingerprint propagation progress.""" - - style: CliStyle = CliStyle(use_color=use_color) - width: int = len(str(total)) - return ( - f" {'fingerprint':<10}" - f"[{completed:>{width}}/{total}] propagated {style.status(status='OK')}" - ) - - -def render_clone_fingerprint_interrupted_line( - *, completed: int, total: int, pending_identities: tuple[str, ...], use_color: bool -) -> str: - """Render bounded context for interrupted clone fingerprint propagation.""" - - style: CliStyle = CliStyle(use_color=use_color) - displayed: tuple[str, ...] = pending_identities[:10] - remaining_count: int = len(pending_identities) - len(displayed) - pending_text: str = ", ".join(displayed) - if remaining_count: - pending_text = f"{pending_text} (+{remaining_count} more)" - return ( - f" {'fingerprint':<10}" - f"[{completed}/{total}] {style.status(status='FAIL')} " - f"not confirmed: {pending_text}" - ) - - -def clone_relation_display_name(relation: str) -> str: - """Normalize ordinary identifiers while preserving quoted identifier case.""" - - parts: list[str] = _QUOTED_IDENTIFIER_PATTERN.split(relation) - return "".join( - part if _QUOTED_IDENTIFIER_PATTERN.fullmatch(part) else part.lower() for part in parts - ) - - -def render_clone_item_line( - *, - index: int, - total: int, - item: CloneItemResult, - use_color: bool, - relation_width: int | None = None, -) -> str: - """Render one streamed clone line: position, action, origin to destination, and status.""" - - style: CliStyle = CliStyle(use_color=use_color) - position: str = f"{index:>{len(str(total))}}/{total}" - plain_relation_flow: str = clone_relation_flow_text( - name=item.name, - origin_relation=item.origin_relation, - destination_relation=item.destination_relation, - ) - relation_flow: str = style.object_name(plain_relation_flow) - if item.origin_relation is not None and item.destination_relation is not None: - relation_flow = ( - f"{style.object_name(clone_relation_display_name(item.origin_relation))} " - f"{style.muted('->')} " - f"{style.object_name(clone_relation_display_name(item.destination_relation))}" - ) - effective_relation_width: int = max(len(plain_relation_flow), relation_width or 0) - relation_padding: str = " " * (effective_relation_width - len(plain_relation_flow)) - action: str = item.action.value - action_width: int = max(len(value.value) for value in CloneAction) - action_padding: str = " " * (action_width - len(action)) - duration: str = ( - f" {style.muted(f'{item.duration_seconds:.2f}s')}" - if item.duration_seconds is not None - else "" - ) - return ( - f" {style.muted(position)} {style.accent(action)}{action_padding} " - f"{relation_flow}{relation_padding} " - f"{style.status(status=_clone_status_text(status=item.status))}{duration}" - ) - - -def render_clone_output( - *, - result: CloneExecutionResult, - elapsed_seconds: float | None = None, - use_color: bool, -) -> None: - style: CliStyle = CliStyle(use_color=use_color) - item: CloneItemResult - for item in result.item_results: - if item.message is not None and item.status != CloneStatus.SUCCESS: - print(f" {style.object_name(item.name)} {style.muted(item.message)}") - success_count: int = sum( - 1 for item in result.item_results if item.status == CloneStatus.SUCCESS - ) - warning_count: int = sum( - 1 for item in result.item_results if item.status == CloneStatus.WARNING - ) - fail_count: int = sum(1 for item in result.item_results if item.status == CloneStatus.FAILED) - cloned_count: int = sum(1 for item in result.item_results if item.action == CloneAction.CLONED) - copied_count: int = sum(1 for item in result.item_results if item.action == CloneAction.COPIED) - recreated_count: int = sum( - 1 for item in result.item_results if item.action == CloneAction.RECREATED_VIEW - ) - recreated_function_count: int = sum( - 1 for item in result.item_results if item.action == CloneAction.RECREATED_FUNCTION - ) - print() - elapsed: str = f"({elapsed_seconds:.2f}s)" if elapsed_seconds is not None else "" - counts_summary: str = format_summary_footer( - counts=( - ("CLONED", cloned_count), - ("COPIED", copied_count), - ("RECREATED_VIEWS", recreated_count), - ("RECREATED_FUNCTIONS", recreated_function_count), - ("PASS", success_count), - ("WARN", warning_count), - ("FAIL", fail_count), - ("TOTAL", len(result.item_results)), - ), - use_color=use_color, - elapsed=elapsed or None, - ) - if warning_count == 0 and fail_count == 0: - completion_state: CompletionState = CompletionState.OK - completion_label: str = "Completed successfully" - elif fail_count == 0: - completion_state = CompletionState.WARN - completion_label = "Completed with warnings" - else: - completion_state = CompletionState.FAIL - completion_label = "Completed with errors" - print( - format_completion_line( - style=style, - state=completion_state, - label=completion_label, - summary=counts_summary, - ) - ) - - -def is_clone_success(result: CloneExecutionResult) -> bool: - return all(item.status != CloneStatus.FAILED for item in result.item_results) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/outputs.py b/src/sqlbuild/cli/commands/_helpers/clone/outputs.py deleted file mode 100644 index 41adf45c2f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/outputs.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Clone command output phases.""" - -from __future__ import annotations - -from sqlbuild._native import format_clone_execution_json, write_execution_json_output -from sqlbuild.cli.commands._helpers.clone.output import ( - is_clone_success, - render_clone_header, - render_clone_output, -) -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneExecutionPreparation, - CloneInvocation, - CloneRunOutcome, -) -from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - FunctionPlanEntry, - ModelPlanEntry, - SeedPlanEntry, -) - - -def write_clone_execution_header( - *, - request: CloneCommandRequest, - invocation: CloneInvocation, - preparation: CloneExecutionPreparation, -) -> None: - """Write the direct clone execution header.""" - - clone_total: int = ( - len(preparation.pipeline_result.destination_source_entries) - + len(preparation.pipeline_result.destination_model_entries) - + len(preparation.pipeline_result.destination_seed_entries) - + len(preparation.pipeline_result.destination_function_entries) - ) - invocation.progress_stream.write( - render_clone_header( - origin_target_name=request.origin_target_name, - destination_target_name=invocation.destination_target_name, - total=clone_total, - use_color=invocation.use_color, - ) - + "\n" - ) - invocation.progress_stream.flush() - - -def write_clone_completion_output(*, invocation: CloneInvocation, outcome: CloneRunOutcome) -> None: - """Render final direct clone command output.""" - - render_clone_output( - result=outcome.result, - elapsed_seconds=outcome.elapsed, - use_color=invocation.use_color, - ) - - -def write_clone_execution_json_output( - *, - request: CloneCommandRequest, - preparation: CloneExecutionPreparation, - outcome: CloneRunOutcome, -) -> None: - """Write structured direct clone outcomes when requested.""" - - entries: tuple[ - CloneSourcePlanEntry | ModelPlanEntry | SeedPlanEntry | FunctionPlanEntry, ... - ] = ( - *preparation.pipeline_result.destination_source_entries, - *preparation.pipeline_result.destination_model_entries, - *preparation.pipeline_result.destination_seed_entries, - *preparation.pipeline_result.destination_function_entries, - ) - resource_types_by_name: dict[str, str] = { - entry.name: str(entry.key.resource_type) for entry in entries - } - write_execution_json_output( - payload=format_clone_execution_json( - result=outcome.result, - resource_types_by_name=resource_types_by_name, - ), - json_output=False, - json_output_path=request.json_output_path, - ) - - -def resolve_clone_exit_code(outcome: CloneRunOutcome) -> int: - """Return the shell exit code for a direct clone outcome.""" - - return 0 if is_clone_success(outcome.result) else 1 diff --git a/src/sqlbuild/cli/commands/_helpers/clone/planning.py b/src/sqlbuild/cli/commands/_helpers/clone/planning.py deleted file mode 100644 index 82b9b6d6c8..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/planning.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Clone command planning phase.""" - -from __future__ import annotations - -import time - -from sqlbuild._native import PlanningProgressReporter -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneConnectionContext, - CloneExecutionPreparation, - CloneInvocation, -) -from sqlbuild.compiler.pipeline.main.clone import run_clone_pipeline -from sqlbuild.compiler.pipeline.models import ClonePipelineConnection, ClonePipelineResult -from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - FunctionPlanEntry, - ModelPlanEntry, - SeedPlanEntry, -) -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def prepare_clone_execution( - *, - request: CloneCommandRequest, - invocation: CloneInvocation, - connection_context: CloneConnectionContext, -) -> CloneExecutionPreparation: - """Prepare the direct clone plan.""" - - progress: PlanningProgressReporter = PlanningProgressReporter( - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - planning_start: float = time.monotonic() - with OperationLifecycle(operation_kind="project", operation_name="project_compile"): - progress.on_progress("Preparing clone plan...") - pipeline_result: ClonePipelineResult = run_clone_pipeline( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - origin_target_name=request.origin_target_name, - destination_target_name=invocation.destination_target_name, - no_sql_validation=request.no_sql_validation, - select=request.select, - exclude=request.exclude, - cli_vars=request.cli_vars, - destination_connection=ClonePipelineConnection( - config=connection_context.destination_connection_config, - handle=connection_context.destination_connection, - ), - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ) - progress.on_progress(f"Prepared clone plan. ({time.monotonic() - planning_start:.2f}s)") - destination_model_entries: tuple[ModelPlanEntry, ...] = ( - pipeline_result.destination_model_entries - ) - destination_seed_entries: tuple[SeedPlanEntry, ...] = pipeline_result.destination_seed_entries - destination_source_entries: tuple[CloneSourcePlanEntry, ...] = ( - pipeline_result.destination_source_entries - ) - destination_function_entries: tuple[FunctionPlanEntry, ...] = ( - pipeline_result.destination_function_entries - ) - if ( - not destination_model_entries - and not destination_seed_entries - and not destination_source_entries - and not destination_function_entries - ): - raise CliUserError("no cloneable resources found in the selected scope", code="C407") - return CloneExecutionPreparation(pipeline_result=pipeline_result) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/validation.py b/src/sqlbuild/cli/commands/_helpers/clone/validation.py deleted file mode 100644 index 8c96cade8d..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/validation.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Clone command validation helpers.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.models import TargetConfig - - -def validate_clone_request( - *, - discovered_inputs: DiscoveredProjectInputs, - origin_target_name: str, - destination_target_name: str, -) -> None: - project_targets: dict[str, TargetConfig] = discovered_inputs.project_config.targets - local_target_names: set[str] = set(discovered_inputs.local_config.targets) - target_names: set[str] = set(project_targets) | local_target_names - if origin_target_name == destination_target_name: - raise CliUserError("clone requires different --from and --to targets", code="C401") - if origin_target_name not in target_names: - raise CliUserError(f"unknown target '{origin_target_name}'", code="C402") - if destination_target_name not in target_names: - raise CliUserError(f"unknown target '{destination_target_name}'", code="C403") - origin_target: TargetConfig = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=origin_target_name, - ) - destination_target: TargetConfig = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=destination_target_name, - ) - if not origin_target.clone.allow_as_clone_origin: - raise CliUserError( - ( - f"target '{origin_target_name}' is not allowed as a clone origin target; " - f"set targets.{origin_target_name}.clone.allow_as_clone_origin = true" - ), - code="C404", - ) - if not destination_target.clone.allow_as_clone_destination: - raise CliUserError( - ( - f"target '{destination_target_name}' is not allowed as a clone destination " - f"target; set targets.{destination_target_name}.clone." - "allow_as_clone_destination = true" - ), - code="C405", - ) diff --git a/src/sqlbuild/cli/commands/_helpers/diff/__init__.py b/src/sqlbuild/cli/commands/_helpers/diff/__init__.py deleted file mode 100644 index b12af6e04c..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Diff command helper package.""" diff --git a/src/sqlbuild/cli/commands/_helpers/diff/evidence.py b/src/sqlbuild/cli/commands/_helpers/diff/evidence.py deleted file mode 100644 index 0ae5378b32..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/evidence.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Bounded, diff-aware rendering for row-diff evidence.""" - -from __future__ import annotations - -import hashlib - -from sqlbuild.cli.commands.models import DiffExampleRenderOptions, RenderedDiffExampleValue - -_MAX_RENDERED_KEY_LENGTH: int = 256 - - -def render_example_pair( - *, - left_value: object, - right_value: object, - options: DiffExampleRenderOptions, -) -> tuple[RenderedDiffExampleValue, RenderedDiffExampleValue]: - """Render two unequal values around their first differing character.""" - - left_text: str = str(left_value) - right_text: str = str(right_value) - first_difference: int | None = _first_difference(left=left_text, right=right_text) - return ( - _render_value(text=left_text, first_difference=first_difference, options=options), - _render_value(text=right_text, first_difference=first_difference, options=options), - ) - - -def display_example_value(value: RenderedDiffExampleValue) -> str: - """Render one bounded value for terminal output.""" - - if value.suppressed: - return f"" - rendered: str = value.text or "" - if value.truncated: - return f"{rendered} [truncated; {value.original_length:,} chars]" - return rendered - - -def render_key_value(value: object) -> str: - """Keep ordinary keys complete and identify exceptionally long shortened keys.""" - - text: str = str(value) - if len(text) <= _MAX_RENDERED_KEY_LENGTH: - return text - digest: str = hashlib.sha256(text.encode("utf-8")).hexdigest()[:12] - return f"{text[:_MAX_RENDERED_KEY_LENGTH]}… [truncated; {len(text):,} chars; sha256:{digest}]" - - -def example_value_payload(value: RenderedDiffExampleValue) -> dict[str, object]: - """Build stable structured evidence for one rendered value.""" - - return { - "value": value.text, - "original_length": value.original_length, - "truncated": value.truncated, - "suppressed": value.suppressed, - "first_difference": value.first_difference, - } - - -def _render_value( - *, - text: str, - first_difference: int | None, - options: DiffExampleRenderOptions, -) -> RenderedDiffExampleValue: - original_length: int = len(text) - if options.suppress_values: - return RenderedDiffExampleValue( - text=None, - original_length=original_length, - truncated=False, - suppressed=True, - first_difference=first_difference, - ) - limit: int | None = options.max_value_length - if limit is None or original_length <= limit: - return RenderedDiffExampleValue( - text=text, - original_length=original_length, - truncated=False, - suppressed=False, - first_difference=first_difference, - ) - difference: int = first_difference or 0 - context_before: int = min(limit // 3, max(limit - 1, 0)) - start: int = min(max(difference - context_before, 0), original_length - limit) - end: int = start + limit - fragment: str = text[start:end] - rendered: str = f"{'…' if start else ''}{fragment}{'…' if end < original_length else ''}" - return RenderedDiffExampleValue( - text=rendered, - original_length=original_length, - truncated=True, - suppressed=False, - first_difference=first_difference, - ) - - -def _first_difference(*, left: str, right: str) -> int | None: - index: int - for index, (left_char, right_char) in enumerate(zip(left, right, strict=False)): - if left_char != right_char: - return index - if len(left) != len(right): - return min(len(left), len(right)) - return None diff --git a/src/sqlbuild/cli/commands/_helpers/diff/execution.py b/src/sqlbuild/cli/commands/_helpers/diff/execution.py deleted file mode 100644 index 81400a38dc..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/execution.py +++ /dev/null @@ -1,721 +0,0 @@ -"""Diff command execution phases.""" - -from __future__ import annotations - -import sys -import time -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - resolve_project_connection_config, - resolve_target_connection_config, - run_diff_pipeline, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.cli.commands._helpers.diff.size_guard import ( - build_diff_size_guard_error, - resolve_full_diff_size_limits, -) -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.constants import READ_ONLY_QUERY_ROOT_KEYS -from sqlbuild.cli.commands.exceptions import CliUserError, QueryDiffExecutionError -from sqlbuild.cli.commands.models import ( - DiffCommandRequest, - DiffInvocation, - DirectDiffPreparation, - QueryDiffPreparation, - QueryDiffRunOutcome, -) -from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config -from sqlbuild.compiler.compile.main.effective_target_namespace import ( - build_effective_target_namespace, -) -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle -from sqlbuild.executor.diff.exceptions import FullDiffSizeGuardError -from sqlbuild.executor.diff.main.config import parse_cli_tolerance_overrides -from sqlbuild.executor.diff.main.execute import execute_diff -from sqlbuild.executor.diff.main.execute_query import ( - execute_query_diff as execute_prepared_query_diff, -) -from sqlbuild.executor.diff.models import ( - DiffExecutionOptions, - DiffExecutionResult, - ModelDiffResult, - QueryDiffArtifact, - QueryDiffArtifactCleanupResult, - RowDiffSamplingOverride, -) -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name - - -def prepare_direct_diff( - *, request: DiffCommandRequest, invocation: DiffInvocation -) -> DirectDiffPreparation: - """Resolve direct target diff adapter, compiled projects, and limits.""" - - if request.from_name is None: - raise CliUserError("model diff requires FROM or FROM:TO", code="C224") - from_target: str = request.from_name - to_target: str = _resolve_diff_to_target( - request=request, invocation=invocation, from_target=from_target - ) - if from_target not in invocation.discovered_inputs.project_config.targets: - raise CliUserError(f"unknown diff FROM target '{from_target}'", code="C205") - if to_target not in invocation.discovered_inputs.project_config.targets: - raise CliUserError(f"unknown diff TO target '{to_target}'", code="C206") - effective_adapter_name: str = resolve_effective_adapter_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=effective_adapter_name, project_dir=invocation.effective_project_dir - ) - connection_config: dict[str, object] = resolve_target_connection_config( - discovered_inputs=invocation.discovered_inputs, - project_dir=invocation.effective_project_dir, - target_name=to_target, - cli_vars=request.cli_vars, - ) - left_project: Any - right_project: Any - selected_names: tuple[str, ...] - left_project, right_project, selected_names = run_diff_pipeline( - discovered_inputs=invocation.discovered_inputs, - adapter=adapter, - from_target=from_target, - to_target=to_target, - resolved_connection=connection_config, - no_sql_validation=request.no_sql_validation, - select=request.select, - exclude=request.exclude, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ) - if not selected_names: - raise CliUserError("no diffable models found in the selected scope", code="C207") - return DirectDiffPreparation( - from_target=from_target, - to_target=to_target, - adapter=adapter, - left_project=left_project, - right_project=right_project, - selected_names=selected_names, - connection_config=connection_config, - effective_max_column_examples=_effective_max_examples( - explicit_value=request.max_column_examples, verbose=request.verbose - ), - effective_max_row_only_examples=_effective_max_examples( - explicit_value=request.max_row_only_examples, verbose=request.verbose - ), - full_size_limits=resolve_full_diff_size_limits( - request=request, - discovered_inputs=invocation.discovered_inputs, - from_target=from_target, - to_target=to_target, - ), - ) - - -def _resolve_diff_to_target( - *, request: DiffCommandRequest, invocation: DiffInvocation, from_target: str -) -> str: - if request.to_name is not None: - return request.to_name - active_target: str | None = resolve_target_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - selected_target=None, - ) - if active_target is None: - raise CliUserError( - "model diff requires FROM:TO when the project has no active target", - code="C271", - help="set default_target, or pass both targets as FROM:TO", - ) - if active_target == from_target: - raise CliUserError( - f"diff FROM '{from_target}' is the active target, so there is nothing to compare", - code="C272", - help="pass both targets as FROM:TO", - ) - return active_target - - -def execute_direct_diff( - *, request: DiffCommandRequest, preparation: DirectDiffPreparation -) -> DiffExecutionResult: - """Execute a direct target-to-target diff.""" - - connection: Any = preparation.adapter.connect(preparation.connection_config) - try: - return execute_diff( - adapter=preparation.adapter, - connection=connection, - left_project=preparation.left_project, - right_project=preparation.right_project, - selected_names=preparation.selected_names, - options=DiffExecutionOptions( - schema_only=request.schema_only, - bounded=request.bounded, - collect_samples=not request.schema_only, - max_column_examples=preparation.effective_max_column_examples, - max_row_only_examples=preparation.effective_max_row_only_examples, - max_models=request.max_models, - max_columns=request.max_columns, - sampling_override=_sampling_override(request=request), - unique_key_override=request.unique_key_override, - unkeyed=request.unkeyed, - excluded_columns_override=request.excluded_columns_override, - tolerance_overrides=parse_cli_tolerance_overrides( - values=request.tolerance_overrides - ), - full_size_limits=preparation.full_size_limits, - ), - ) - except FullDiffSizeGuardError as error: - raise build_diff_size_guard_error( - request=request, - from_target=preparation.from_target, - to_target=preparation.to_target, - error=error, - ) from error - finally: - preparation.adapter.close(connection) - - -def prepare_query_diff( - *, request: DiffCommandRequest, invocation: DiffInvocation -) -> QueryDiffPreparation: - """Resolve raw SQL, active target namespace, adapter, and connection.""" - - effective_adapter_name: str = resolve_effective_adapter_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=effective_adapter_name, - project_dir=invocation.effective_project_dir, - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=invocation.discovered_inputs, - project_dir=invocation.effective_project_dir, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - connection_database: object | None = connection_config.get("database") - if effective_adapter_name in { - BuiltinAdapter.DUCKDB.value, - BuiltinAdapter.MOTHERDUCK.value, - }: - connection_database = None - target_name, _target, database, schema = build_effective_target_namespace( - discovered_inputs=invocation.discovered_inputs, - selected_target=request.selected_target, - connection_database=connection_database, - connection_schema=connection_config.get("schema"), - default_database=adapter.default_database(), - default_schema=adapter.default_schema(), - ) - if schema is None: - raise CliUserError( - "raw-query diff requires an active target schema", - code="C227", - help="configure the selected target schema before running the comparison", - ) - left_sql: str = _resolve_query_input( - inline=request.left_query, - file=request.left_query_file, - label="left", - ) - right_sql: str = _resolve_query_input( - inline=request.right_query, - file=request.right_query_file, - label="right", - ) - left_sql = _validate_read_only_query(sql=left_sql, label="left", adapter=adapter) - right_sql = _validate_read_only_query(sql=right_sql, label="right", adapter=adapter) - _runtime_target, _effective_vars, run_id = build_effective_runtime_config( - discovered_inputs=invocation.discovered_inputs, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - return QueryDiffPreparation( - adapter=adapter, - connection_config=connection_config, - left_sql=left_sql, - right_sql=right_sql, - left_label=(request.left_label or "left query").strip(), - right_label=(request.right_label or "right query").strip(), - selected_target=target_name, - database=database, - schema=schema, - run_id=run_id, - artifact_ttl=invocation.discovered_inputs.project_config.diff.query_artifact_ttl, - effective_max_column_examples=_effective_max_examples( - explicit_value=request.max_column_examples, - verbose=request.verbose, - ), - effective_max_row_only_examples=_effective_max_examples( - explicit_value=request.max_row_only_examples, - verbose=request.verbose, - ), - ) - - -def execute_query_diff( - *, request: DiffCommandRequest, preparation: QueryDiffPreparation -) -> QueryDiffRunOutcome: - """Materialize, compare, and clean one raw-query pair.""" - - adapter: BaseAdapter = preparation.adapter - total_started_at: float = time.perf_counter() - phase_seconds: dict[str, float] = {} - print("Query diff connection START", file=sys.stderr) - connection_started_at: float = time.perf_counter() - try: - connection: Any = adapter.connect(preparation.connection_config) - except Exception: - connection_elapsed: float = time.perf_counter() - connection_started_at - print(f"Query diff connection ERROR ({connection_elapsed:.2f}s)", file=sys.stderr) - print( - f"Query diff total ERROR ({time.perf_counter() - total_started_at:.2f}s)", - file=sys.stderr, - ) - raise - phase_seconds["connection"] = time.perf_counter() - connection_started_at - print(f"Query diff connection OK ({phase_seconds['connection']:.2f}s)", file=sys.stderr) - left, right = _build_query_diff_artifacts(adapter=adapter, preparation=preparation) - created: list[QueryDiffArtifact] = [] - cleanup_errors: list[Exception] = [] - model_result: ModelDiffResult | None = None - try: - phase_seconds["reconciliation"] = _reconcile_query_diff_artifacts( - adapter=adapter, - connection=connection, - preparation=preparation, - ) - expires_at, phase_seconds["inspection"] = _inspect_query_diff_inputs( - adapter=adapter, - connection=connection, - preparation=preparation, - artifacts=(left, right), - max_columns=request.max_columns, - ) - print("Query diff materialization START", file=sys.stderr) - materialization_started_at: float = time.perf_counter() - try: - QueryDiffArtifactLifecycle.materialize( - adapter=adapter, - connection=connection, - artifact=left, - sql=preparation.left_sql, - expires_at=expires_at, - ) - created.append(left) - QueryDiffArtifactLifecycle.materialize( - adapter=adapter, - connection=connection, - artifact=right, - sql=preparation.right_sql, - expires_at=expires_at, - ) - created.append(right) - except Exception: - phase_seconds["materialization"] = time.perf_counter() - materialization_started_at - print( - f"Query diff materialization ERROR ({phase_seconds['materialization']:.2f}s)", - file=sys.stderr, - ) - raise - phase_seconds["materialization"] = time.perf_counter() - materialization_started_at - print( - f"Query diff materialization OK ({preparation.schema}; expires " - f"{expires_at.isoformat()}; {phase_seconds['materialization']:.2f}s)", - file=sys.stderr, - ) - print("Query diff comparison START", file=sys.stderr) - comparison_started_at: float = time.perf_counter() - try: - model_result = execute_prepared_query_diff( - adapter=adapter, - connection=connection, - left_relation=left.relation, - right_relation=right.relation, - options=DiffExecutionOptions( - schema_only=request.schema_only, - collect_samples=not request.schema_only, - max_column_examples=preparation.effective_max_column_examples, - max_row_only_examples=preparation.effective_max_row_only_examples, - max_columns=request.max_columns, - sampling_override=_sampling_override(request=request), - unique_key_override=request.unique_key_override, - unkeyed=request.unkeyed, - excluded_columns_override=request.excluded_columns_override, - tolerance_overrides=parse_cli_tolerance_overrides( - values=request.tolerance_overrides - ), - comparison_name=(f"{preparation.left_label} vs {preparation.right_label}"), - ), - ) - except Exception: - phase_seconds["comparison"] = time.perf_counter() - comparison_started_at - print( - f"Query diff comparison ERROR ({phase_seconds['comparison']:.2f}s)", - file=sys.stderr, - ) - raise - phase_seconds["comparison"] = time.perf_counter() - comparison_started_at - print(f"Query diff comparison OK ({phase_seconds['comparison']:.2f}s)", file=sys.stderr) - finally: - active_error: BaseException | None = sys.exc_info()[1] - cleanup_errors, phase_seconds["cleanup"] = _cleanup_query_diff_artifacts( - adapter=adapter, - connection=connection, - created=created, - ) - phase_seconds["total"] = time.perf_counter() - total_started_at - total_status: str = "ERROR" if active_error is not None or cleanup_errors else "OK" - print( - f"Query diff total {total_status} ({phase_seconds['total']:.2f}s)", - file=sys.stderr, - ) - if cleanup_errors: - raise cleanup_errors[0] - if model_result is None: - raise QueryDiffExecutionError( - "query diff comparison completed without a result", code="C256" - ) - result: DiffExecutionResult = DiffExecutionResult(model_results=(model_result,)) - outcome, exit_code = _resolve_query_diff_outcome(request=request, result=result) - return QueryDiffRunOutcome( - result=result, - phase_seconds=phase_seconds, - outcome=outcome, - exit_code=exit_code, - ) - - -def _build_query_diff_artifacts( - *, adapter: BaseAdapter, preparation: QueryDiffPreparation -) -> tuple[QueryDiffArtifact, QueryDiffArtifact]: - return ( - QueryDiffArtifactLifecycle.build( - adapter=adapter, - run_id=preparation.run_id, - side="left", - database=preparation.database, - schema=preparation.schema, - ), - QueryDiffArtifactLifecycle.build( - adapter=adapter, - run_id=preparation.run_id, - side="right", - database=preparation.database, - schema=preparation.schema, - ), - ) - - -def _cleanup_query_diff_artifacts( - *, - adapter: BaseAdapter, - connection: Any, - created: list[QueryDiffArtifact], -) -> tuple[list[Exception], float]: - print("Query diff immediate cleanup START", file=sys.stderr) - started_at: float = time.perf_counter() - errors: list[Exception] = [] - for artifact in reversed(created): - try: - QueryDiffArtifactLifecycle.cleanup( - adapter=adapter, - connection=connection, - artifact=artifact, - ) - except Exception as error: - errors.append(error) - try: - adapter.close(connection) - except Exception as error: - errors.append(error) - elapsed: float = time.perf_counter() - started_at - if errors: - print( - f"Query diff immediate cleanup ERROR ({len(errors)} failed; {elapsed:.2f}s)", - file=sys.stderr, - ) - else: - print( - f"Query diff immediate cleanup OK ({len(created)} removed; {elapsed:.2f}s)", - file=sys.stderr, - ) - return errors, elapsed - - -def _resolve_query_diff_outcome( - *, request: DiffCommandRequest, result: DiffExecutionResult -) -> tuple[str, int]: - if not request.schema_only and _query_schema_prevented_row_comparison(result=result): - return "incomplete", 2 - if _query_result_has_findings(result=result): - return "findings", 1 - return "pass", 0 - - -def _query_schema_prevented_row_comparison(*, result: DiffExecutionResult) -> bool: - return any( - model.row_result is None - and ( - model.schema_result.added_columns - or model.schema_result.removed_columns - or model.schema_result.type_changed_columns - ) - for model in result.model_results - ) - - -def _query_result_has_findings(*, result: DiffExecutionResult) -> bool: - return any( - model.schema_result.added_columns - or model.schema_result.removed_columns - or model.schema_result.type_changed_columns - or ( - model.row_result is not None - and ( - model.row_result.unequal_count - or model.row_result.left_only_count - or model.row_result.right_only_count - ) - ) - for model in result.model_results - ) - - -def _inspect_query_diff_inputs( - *, - adapter: BaseAdapter, - connection: Any, - preparation: QueryDiffPreparation, - artifacts: tuple[QueryDiffArtifact, QueryDiffArtifact], - max_columns: int | None, -) -> tuple[datetime, float]: - print("Query diff inspection START", file=sys.stderr) - started_at: float = time.perf_counter() - try: - artifact_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple( - (artifact.database, artifact.schema, artifact.name) for artifact in artifacts - ), - ) - for artifact in artifacts: - if artifact_lookup.exists( - database=artifact.database, - schema=artifact.schema, - name=artifact.name, - ): - raise CliUserError( - f"query-diff artifact already exists: {artifact.relation}", - code="C245", - help="wait for its TTL cleanup or investigate the reserved-name relation", - ) - ttl: Duration | None = Duration.parse(preparation.artifact_ttl) - if ttl is None: - raise CliUserError("diff.query_artifact_ttl must be a positive duration", code="C228") - _preflight_query_columns( - adapter=adapter, - connection=connection, - sql=preparation.left_sql, - label=preparation.left_label, - max_columns=max_columns, - ) - _preflight_query_columns( - adapter=adapter, - connection=connection, - sql=preparation.right_sql, - label=preparation.right_label, - max_columns=max_columns, - ) - except Exception: - elapsed: float = time.perf_counter() - started_at - print( - f"Query diff inspection ERROR ({elapsed:.2f}s)", - file=sys.stderr, - ) - raise - elapsed = time.perf_counter() - started_at - print(f"Query diff inspection OK ({elapsed:.2f}s)", file=sys.stderr) - return ttl.add_to(datetime.now(UTC)), elapsed - - -def _reconcile_query_diff_artifacts( - *, - adapter: BaseAdapter, - connection: Any, - preparation: QueryDiffPreparation, -) -> float: - print("Query diff reconciliation START", file=sys.stderr) - started_at: float = time.perf_counter() - try: - result: QueryDiffArtifactCleanupResult = QueryDiffArtifactLifecycle.cleanup_expired( - adapter=adapter, - connection=connection, - database=preparation.database, - schema=preparation.schema, - now=datetime.now(UTC), - ) - except Exception as error: - elapsed: float = time.perf_counter() - started_at - print( - "Query diff reconciliation WARNING recovery cleanup could not inspect historical " - f"ownership evidence: {error} ({elapsed:.2f}s)", - file=sys.stderr, - ) - return elapsed - elapsed = time.perf_counter() - started_at - print( - f"Query diff reconciliation OK ({len(result.cleaned)} expired; {elapsed:.2f}s)", - file=sys.stderr, - ) - for relation in result.untracked: - print( - "Query diff reconciliation WARNING reserved-name relation lacks matching " - f"ownership evidence: {relation.name}", - file=sys.stderr, - ) - return elapsed - - -def _resolve_query_input(*, inline: str | None, file: Path | None, label: str) -> str: - if inline is not None: - return inline - if file is None: - raise CliUserError(f"raw-query diff requires a {label} query", code="C229") - if not file.exists() or not file.is_file(): - raise CliUserError(f"{label} query file does not exist: {file}", code="C230") - try: - return file.read_text(encoding="utf-8") - except (OSError, UnicodeError) as error: - raise CliUserError( - f"{label} query file could not be read as UTF-8: {file}", code="C231" - ) from error - - -def _validate_read_only_query(*, sql: str, label: str, adapter: BaseAdapter) -> str: - normalized_sql: str = sql.strip() - if normalized_sql.endswith(";"): - normalized_sql = normalized_sql[:-1].rstrip() - if not normalized_sql: - raise CliUserError(f"{label} query must not be empty", code="C232") - polyglot: Any | None = import_polyglot_sql() - if polyglot is None: - raise CliUserError("raw-query diff requires the bundled SQL parser", code="C233") - try: - expression: Any = polyglot.parse_one( - normalized_sql, - dialect=adapter.sql_analysis_dialect() or "generic", - ) - except Exception as error: - raise CliUserError( - f"{label} query is not one valid SQL statement: {error}", code="C234" - ) from error - if str(getattr(expression, "key", "")).lower() not in READ_ONLY_QUERY_ROOT_KEYS: - raise CliUserError( - f"{label} query must be one read-only SELECT, WITH, or VALUES expression", - code="C235", - ) - mutating_keys: frozenset[str] = frozenset( - { - "alter", - "command", - "copy", - "create", - "delete", - "drop", - "execute", - "grant", - "insert", - "load_data", - "merge", - "pragma", - "revoke", - "set", - "transaction", - "truncate", - "uncache", - "update", - "use", - } - ) - if any(str(getattr(node, "key", "")).lower() in mutating_keys for node in expression.walk()): - raise CliUserError( - f"{label} query must not contain data-changing or administrative statements", - code="C240", - ) - expression_args: object = getattr(expression, "args", {}) - if isinstance(expression_args, dict) and ( - expression_args.get("into") is not None or bool(expression_args.get("locks")) - ): - raise CliUserError( - f"{label} query must not write a relation or acquire row locks", - code="C244", - ) - return normalized_sql - - -def _preflight_query_columns( - *, - adapter: BaseAdapter, - connection: Any, - sql: str, - label: str, - max_columns: int | None, -) -> None: - display_label: str = label if label.lower().endswith("query") else f"{label} query" - columns: tuple[str, ...] = adapter.query_column_names(connection=connection, sql=sql) - if not columns: - raise CliUserError(f"{display_label} returned no columns", code="C237") - normalized: tuple[str, ...] = tuple(column.lower() for column in columns) - duplicates: tuple[str, ...] = tuple( - sorted({column for column in normalized if normalized.count(column) > 1}) - ) - if duplicates: - raise CliUserError( - f"{display_label} returns duplicate column names: {', '.join(duplicates)}", - code="C238", - help="add unique aliases before comparing query results", - ) - if max_columns is not None and len(columns) > max_columns: - raise CliUserError( - f"{display_label} has {len(columns)} columns, exceeding --max-columns {max_columns}", - code="C239", - ) - - -def _effective_max_examples(*, explicit_value: int | None, verbose: bool) -> int: - return explicit_value if explicit_value is not None else (10 if verbose else 3) - - -def _sampling_override(*, request: DiffCommandRequest) -> RowDiffSamplingOverride: - return RowDiffSamplingOverride( - row_limit=request.sample_rows, - seed=request.sample_seed, - exhaustive=request.exhaustive, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/diff/invocation.py b/src/sqlbuild/cli/commands/_helpers/diff/invocation.py deleted file mode 100644 index fce0894b86..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/invocation.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Diff command invocation resolution and validation.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import discover_project_inputs -from sqlbuild.cli.commands.exceptions import CliUserError, QueryDiffIncompleteError -from sqlbuild.cli.commands.models import DiffCommandRequest, DiffInvocation -from sqlbuild.compiler.discovery.main.discover_configuration import ( - discover_project_configuration, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors - - -def resolve_diff_invocation(*, request: DiffCommandRequest) -> DiffInvocation: - """Validate diff flags and discover project inputs.""" - - try: - _validate_diff_request(request=request) - except CliUserError as error: - if is_query_diff_request(request=request): - raise QueryDiffIncompleteError( - error.message, - code=error.code, - help=error.help, - ) from error - raise - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = ( - discover_project_configuration(project_dir=effective_project_dir) - if is_query_diff_request(request=request) - else discover_project_inputs(project_dir=effective_project_dir) - ) - reject_unit_test_selectors( - select=request.select, exclude=request.exclude, discovered_inputs=discovered_inputs - ) - if not request.select and not is_query_diff_request(request=request): - raise CliUserError("diff requires --select in v1", code="C204") - return DiffInvocation( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - ) - - -def _validate_diff_request(*, request: DiffCommandRequest) -> None: - query_mode: bool = is_query_diff_request(request=request) - selected_modes: int = ( - int(request.full) + int(request.schema_only) + int(request.bounded is not None) - ) - if selected_modes > 1: - raise CliUserError( - "diff accepts at most one of --full, --schema-only, or --bounded", - code="C201", - ) - if request.max_column_examples is not None and request.max_column_examples <= 0: - raise CliUserError("diff --max-column-examples must be positive", code="C202") - if request.max_row_only_examples is not None and request.max_row_only_examples <= 0: - raise CliUserError("diff --max-row-only-examples must be positive", code="C203") - if request.sample_rows is not None and request.sample_rows <= 0: - raise CliUserError("diff --sample-rows must be positive", code="C208") - if request.exhaustive and request.sample_rows is not None: - raise CliUserError("diff --exhaustive cannot be combined with --sample-rows", code="C209") - if request.max_models is not None and request.max_models <= 0: - raise CliUserError("diff --max-models must be positive", code="C210") - if request.max_columns is not None and request.max_columns <= 0: - raise CliUserError("diff --max-columns must be positive", code="C211") - if request.max_value_length is not None and request.max_value_length <= 0: - raise CliUserError("diff --max-value-length must be positive", code="C246") - if request.full_example_values and request.max_value_length is not None: - raise CliUserError( - "diff --full-example-values cannot be combined with --max-value-length", - code="C247", - ) - if request.full_example_values and request.suppress_example_values: - raise CliUserError( - "diff --full-example-values cannot be combined with --no-example-values", - code="C248", - ) - if request.unkeyed and request.unique_key_override: - raise CliUserError("diff --unkeyed cannot be combined with --key", code="C226") - if request.unkeyed and (request.sample_rows is not None or request.sample_seed is not None): - raise CliUserError("unkeyed diff does not support sampling", code="C222") - if request.unkeyed and request.tolerance_overrides: - raise CliUserError("unkeyed diff does not support tolerances", code="C223") - if any(not key.strip() for key in request.unique_key_override): - raise CliUserError("diff --key values must not be empty", code="C241") - normalized_keys: tuple[str, ...] = tuple(key.lower() for key in request.unique_key_override) - if len(set(normalized_keys)) != len(normalized_keys): - raise CliUserError("diff --key values must be unique", code="C242") - if query_mode: - _validate_query_diff_request(request=request) - else: - _validate_model_diff_request(request=request) - - -def is_query_diff_request(*, request: DiffCommandRequest) -> bool: - """Return whether any raw-query input was supplied.""" - - return any( - value is not None - for value in ( - request.left_query, - request.left_query_file, - request.right_query, - request.right_query_file, - ) - ) - - -def _validate_query_diff_request(*, request: DiffCommandRequest) -> None: - if request.from_name is not None or request.to_name is not None: - raise CliUserError("raw-query diff does not accept FROM:TO", code="C212") - if request.select or request.exclude: - raise CliUserError("raw-query diff does not accept model selectors", code="C213") - if request.bounded is not None: - raise CliUserError( - "raw-query diff does not support --bounded; filter both raw queries explicitly", - code="C214", - ) - if request.max_models is not None: - raise CliUserError("raw-query diff does not accept --max-models", code="C243") - if request.left_label is not None and not request.left_label.strip(): - raise CliUserError("raw-query diff --left-label must not be empty", code="C249") - if request.right_label is not None and not request.right_label.strip(): - raise CliUserError("raw-query diff --right-label must not be empty", code="C250") - if request.left_query is None and request.left_query_file is None: - raise CliUserError("raw-query diff requires a left query", code="C216") - if request.left_query is not None and request.left_query_file is not None: - raise CliUserError( - "raw-query diff accepts either --left-query or --left-query-file, not both", - code="C217", - ) - if request.right_query is None and request.right_query_file is None: - raise CliUserError("raw-query diff requires a right query", code="C218") - if request.right_query is not None and request.right_query_file is not None: - raise CliUserError( - "raw-query diff accepts either --right-query or --right-query-file, not both", - code="C219", - ) - if request.schema_only: - if request.unkeyed or request.unique_key_override: - raise CliUserError( - "schema-only raw-query diff does not accept --key or --unkeyed", - code="C220", - ) - return - if bool(request.unique_key_override) == request.unkeyed: - raise CliUserError( - "raw-query diff requires one or more --key values or explicit --unkeyed", - code="C221", - ) - - -def _validate_model_diff_request(*, request: DiffCommandRequest) -> None: - if request.from_name is None: - raise CliUserError("model diff requires FROM or FROM:TO", code="C224") - if request.selected_target is not None: - raise CliUserError("model diff does not accept --target", code="C225") - if request.left_label is not None or request.right_label is not None: - raise CliUserError("model diff does not accept query labels", code="C251") diff --git a/src/sqlbuild/cli/commands/_helpers/diff/json_output.py b/src/sqlbuild/cli/commands/_helpers/diff/json_output.py deleted file mode 100644 index c3c872d1bd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/json_output.py +++ /dev/null @@ -1,314 +0,0 @@ -"""Structured JSON output for diff results.""" - -from __future__ import annotations - -import json -import math -from datetime import date, datetime -from pathlib import Path - -from sqlbuild.adapter.contract.models import RowDiffCoverage, RowDiffResult, RowDiffSampleRow -from sqlbuild.cli.commands._helpers.diff.evidence import ( - example_value_payload, - render_example_pair, -) -from sqlbuild.cli.commands._helpers.diff.output import has_diff_failures -from sqlbuild.cli.commands.constants import DEFAULT_DIFF_MAX_VALUE_LENGTH -from sqlbuild.cli.commands.models import DiffExampleRenderOptions -from sqlbuild.executor.diff.constants import DIFF_INPUT_KIND_QUERY -from sqlbuild.executor.diff.models import DiffExecutionResult, ModelDiffResult - -_DEFAULT_EXAMPLE_RENDER_OPTIONS: DiffExampleRenderOptions = DiffExampleRenderOptions( - max_value_length=DEFAULT_DIFF_MAX_VALUE_LENGTH -) - - -def render_diff_json_output( - *, - result: DiffExecutionResult, - from_label: str, - to_label: str, - example_render_options: DiffExampleRenderOptions = _DEFAULT_EXAMPLE_RENDER_OPTIONS, - phase_seconds: dict[str, float] | None = None, - outcome: str | None = None, -) -> str: - """Render one stable structured diff document.""" - - return json.dumps( - _json_safe_value( - _diff_payload( - result=result, - from_label=from_label, - to_label=to_label, - example_render_options=example_render_options, - phase_seconds=phase_seconds, - outcome=outcome, - ) - ), - allow_nan=False, - indent=2, - sort_keys=True, - default=_json_default, - ) - - -def render_diff_error_json( - *, - status: str, - code: str, - message: str, - help_text: str | None = None, - size_guard: dict[str, object] | None = None, -) -> str: - """Render a machine-readable incomplete or execution-failure outcome.""" - - error: dict[str, object] = {"code": code, "message": message} - if help_text is not None: - error["help"] = help_text - payload: dict[str, object] = { - "schema_version": 1, - "status": status, - "outcome": status, - "error": error, - "models": [], - "query_comparisons": [], - } - if size_guard is not None: - payload["size_guard"] = size_guard - return json.dumps(payload, indent=2, sort_keys=True) - - -def _json_safe_value(value: object) -> object: - if isinstance(value, float) and not math.isfinite(value): - if math.isnan(value): - label: str = "NaN" - elif value > 0: - label = "Infinity" - else: - label = "-Infinity" - return {"kind": "non_finite_number", "value": label} - if isinstance(value, dict): - return {key: _json_safe_value(item) for key, item in value.items()} - if isinstance(value, list | tuple): - return [_json_safe_value(item) for item in value] - return value - - -def write_diff_json_output( - *, - path: Path | None, - result: DiffExecutionResult, - from_label: str, - to_label: str, - example_render_options: DiffExampleRenderOptions = _DEFAULT_EXAMPLE_RENDER_OPTIONS, - phase_seconds: dict[str, float] | None = None, - outcome: str | None = None, -) -> None: - """Write a stable structured diff document when a path is requested.""" - - if path is None: - return - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - render_diff_json_output( - result=result, - from_label=from_label, - to_label=to_label, - example_render_options=example_render_options, - phase_seconds=phase_seconds, - outcome=outcome, - ) - + "\n", - encoding="utf-8", - ) - - -def _diff_payload( - *, - result: DiffExecutionResult, - from_label: str, - to_label: str, - example_render_options: DiffExampleRenderOptions, - phase_seconds: dict[str, float] | None, - outcome: str | None, -) -> dict[str, object]: - model_payloads: list[dict[str, object]] = [ - _model_payload( - model=model, - from_label=from_label, - to_label=to_label, - example_render_options=example_render_options, - ) - for model in result.model_results - ] - resolved_outcome: str = outcome or ("findings" if has_diff_failures(result) else "pass") - status: str = { - "pass": "no_differences_found", - "findings": "differences_found", - }.get(resolved_outcome, resolved_outcome) - payload: dict[str, object] = { - "schema_version": 1, - "status": status, - "outcome": resolved_outcome, - "from": from_label, - "to": to_label, - "models": model_payloads, - } - query_payloads: list[dict[str, object]] = [ - model_payload - for model, model_payload in zip(result.model_results, model_payloads, strict=True) - if model.input_kind == DIFF_INPUT_KIND_QUERY - ] - if query_payloads: - payload["input_kind"] = DIFF_INPUT_KIND_QUERY - payload["query_comparisons"] = query_payloads - if phase_seconds is not None: - payload["phase_seconds"] = { - phase: round(seconds, 6) for phase, seconds in sorted(phase_seconds.items()) - } - return payload - - -def _model_payload( - *, - model: ModelDiffResult, - from_label: str, - to_label: str, - example_render_options: DiffExampleRenderOptions, -) -> dict[str, object]: - rows: RowDiffResult | None = model.row_result - payload: dict[str, object] = { - "input_kind": model.input_kind, - "name": model.name, - "from_label": from_label, - "to_label": to_label, - "from_relation": model.left_relation, - "to_relation": model.right_relation, - "unique_key": list(model.unique_key), - "unkeyed": model.unkeyed, - "excluded_columns": list(model.excluded_columns), - "schema": { - "added_columns": [column.name for column in model.schema_result.added_columns], - "removed_columns": [column.name for column in model.schema_result.removed_columns], - "type_changed_columns": [ - {"name": left.name, "from_type": left.type, "to_type": right.type} - for left, right in model.schema_result.type_changed_columns - ], - }, - } - if rows is None: - payload["comparison_scope"] = "schema_only" - return payload - payload["comparison_scope"] = "exhaustive" if rows.is_exhaustive else "sampled" - payload["coverage"] = { - "requested": { - "cursor_column": rows.cursor_column, - "start": rows.cursor_start, - "end": rows.cursor_end, - }, - "from": _coverage_payload(coverage=rows.left_coverage), - "to": _coverage_payload(coverage=rows.right_coverage), - "differs": _coverage_differs(rows=rows), - } - payload["sampling"] = { - "configured_row_limit": rows.sampling.row_limit if rows.sampling is not None else None, - "seed": rows.sampling.seed if rows.sampling is not None else None, - "population_keys": None if model.unkeyed else rows.population_count, - "compared_keys": None if model.unkeyed else rows.compared_count, - "coverage_ratio": ( - rows.compared_count / rows.population_count if rows.population_count else 1.0 - ), - "exhaustive": rows.is_exhaustive, - } - payload["rows"] = { - "from_count": rows.left_count, - "to_count": rows.right_count, - "joined_count": rows.equal_count + rows.unequal_count, - "equal_count": rows.equal_count, - "unequal_count": rows.unequal_count, - "from_only_count": rows.left_only_count, - "to_only_count": rows.right_only_count, - "changed_columns": {column.name: column.mismatched_count for column in rows.column_results}, - "column_mismatches": [ - {"column": column.name, "rows": column.mismatched_count} - for column in rows.column_results - if column.mismatched_count > 0 - ], - } - payload["examples"] = _examples_payload( - model=model, - options=example_render_options, - ) - return payload - - -def _examples_payload( - *, model: ModelDiffResult, options: DiffExampleRenderOptions -) -> dict[str, object]: - unequal_rows: list[dict[str, object]] = [] - values_truncated: bool = False - sample_row: RowDiffSampleRow - for sample_row in model.unequal_row_samples: - cells: list[dict[str, object]] = [] - for cell in sample_row.changed_cells: - left_value, right_value = render_example_pair( - left_value=cell.left_value, - right_value=cell.right_value, - options=options, - ) - values_truncated = values_truncated or left_value.truncated or right_value.truncated - cells.append( - { - "column": cell.name, - "from": example_value_payload(left_value), - "to": example_value_payload(right_value), - } - ) - unequal_rows.append( - { - "key": {name: value for name, value in sample_row.key_values}, - "changed_cells": cells, - } - ) - rows: RowDiffResult | None = model.row_result - return { - "unequal_rows": unequal_rows, - "from_only_keys": [dict(sample) for sample in model.left_only_key_samples], - "to_only_keys": [dict(sample) for sample in model.right_only_key_samples], - "truncated_by_count": bool( - rows is not None - and ( - rows.unequal_count > len(model.unequal_row_samples) - or rows.left_only_count > len(model.left_only_key_samples) - or rows.right_only_count > len(model.right_only_key_samples) - ) - ), - "values_truncated": values_truncated, - "values_suppressed": options.suppress_values, - "max_value_length": options.max_value_length, - } - - -def _coverage_payload(*, coverage: RowDiffCoverage | None) -> dict[str, object] | None: - if coverage is None: - return None - return { - "row_count": coverage.row_count, - "minimum_cursor": coverage.minimum_cursor, - "maximum_cursor": coverage.maximum_cursor, - } - - -def _coverage_differs(*, rows: RowDiffResult) -> bool: - if rows.cursor_column is None or rows.left_coverage is None or rows.right_coverage is None: - return False - return ( - rows.left_coverage.minimum_cursor != rows.right_coverage.minimum_cursor - or rows.left_coverage.maximum_cursor != rows.right_coverage.maximum_cursor - ) - - -def _json_default(value: object) -> str: - if isinstance(value, datetime | date): - return value.isoformat() - return str(value) diff --git a/src/sqlbuild/cli/commands/_helpers/diff/output.py b/src/sqlbuild/cli/commands/_helpers/diff/output.py deleted file mode 100644 index b235670339..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/output.py +++ /dev/null @@ -1,652 +0,0 @@ -"""Output helpers for the diff command.""" - -from __future__ import annotations - -from rich import box -from rich.columns import Columns as RichColumns -from rich.console import Console, Group, RenderableType -from rich.panel import Panel -from rich.table import Table -from rich.text import Text - -from sqlbuild.adapter.contract.models import ( - RowDiffColumnResult, - RowDiffCoverage, - RowDiffResult, - RowDiffSampleCell, - RowDiffSampleRow, - SchemaDiffResult, -) -from sqlbuild.cli.commands._helpers.diff.evidence import ( - display_example_value, - render_example_pair, - render_key_value, -) -from sqlbuild.cli.commands.constants import DEFAULT_DIFF_MAX_VALUE_LENGTH -from sqlbuild.cli.commands.models import DiffExampleRenderOptions -from sqlbuild.executor.diff.constants import DIFF_INPUT_KIND_QUERY -from sqlbuild.executor.diff.models import DiffExecutionResult, ModelDiffResult - -_WIDTH: int = 110 -_TOP_CHANGED_COLUMNS: int = 5 -_RICH_TITLE_STYLE: str = "bold" -_RICH_OBJECT_STYLE: str = "bold cyan" -_RICH_SECTION_STYLE: str = "bold" -_RICH_ADDED_STYLE: str = "green" -_RICH_REMOVED_STYLE: str = "red" -_RICH_MUTED_STYLE: str = "dim" -_DEFAULT_EXAMPLE_RENDER_OPTIONS: DiffExampleRenderOptions = DiffExampleRenderOptions( - max_value_length=DEFAULT_DIFF_MAX_VALUE_LENGTH -) - - -def render_diff_output( - *, - result: DiffExecutionResult, - from_label: str, - to_label: str, - mode_label: str, - use_color: bool, - verbose: bool, - max_column_examples: int, - max_row_only_examples: int, - example_render_options: DiffExampleRenderOptions = _DEFAULT_EXAMPLE_RENDER_OPTIONS, - outcome: str | None = None, -) -> str: - """Render a concise Rich terminal summary for a diff result.""" - - console: Console = Console(force_terminal=use_color, color_system="auto", width=_WIDTH) - with console.capture() as capture: - if not result.model_results: - console.print("No models selected for diff.") - else: - console.print( - Panel( - Text("SQLBuild Diff Summary", style=_RICH_TITLE_STYLE, justify="center"), - box=box.HEAVY, - ) - ) - console.print( - Text(f"{from_label} vs {to_label}", style=_RICH_OBJECT_STYLE, justify="center") - ) - input_label: str = ( - "query pairs" - if all(item.input_kind == DIFF_INPUT_KIND_QUERY for item in result.model_results) - else "models" - ) - console.print( - Text(f"selected {input_label}: {len(result.model_results):,}", justify="center") - ) - resolved_outcome: str = outcome or ("findings" if has_diff_failures(result) else "pass") - console.print(Text(f"outcome: {resolved_outcome}", justify="center")) - console.print() - model_result: ModelDiffResult - for index, model_result in enumerate(result.model_results): - if index > 0: - console.print() - console.print(Text("─" * _WIDTH, style=_RICH_MUTED_STYLE)) - console.print() - _render_model_result( - console=console, - model_result=model_result, - from_label=from_label, - to_label=to_label, - mode_label=mode_label, - verbose=verbose, - max_column_examples=max_column_examples, - max_row_only_examples=max_row_only_examples, - example_render_options=example_render_options, - ) - return capture.get().rstrip() - - -def has_diff_failures(result: DiffExecutionResult) -> bool: - """Return true when any selected model has schema or row differences.""" - - model_result: ModelDiffResult - for model_result in result.model_results: - schema_result: SchemaDiffResult = model_result.schema_result - if ( - schema_result.added_columns - or schema_result.removed_columns - or schema_result.type_changed_columns - ): - return True - row_result: RowDiffResult | None = model_result.row_result - if row_result is not None and ( - row_result.unequal_count or row_result.left_only_count or row_result.right_only_count - ): - return True - return False - - -def _render_model_result( - *, - console: Console, - model_result: ModelDiffResult, - from_label: str, - to_label: str, - mode_label: str, - verbose: bool, - max_column_examples: int, - max_row_only_examples: int, - example_render_options: DiffExampleRenderOptions, -) -> None: - console.print( - _render_overview( - model_result=model_result, - mode_label=mode_label, - ) - ) - console.print() - _print_section(console=console, title="Schemas", content=_render_schema_summary(model_result)) - if model_result.row_result is not None: - _print_section( - console=console, - title="Coverage", - content=_render_coverage( - rows=model_result.row_result, - from_label=from_label, - to_label=to_label, - ), - ) - _print_section( - console=console, - title="Rows" if model_result.row_result.is_exhaustive else "Sampled Rows", - content=_render_rows( - rows=model_result.row_result, - from_label=from_label, - to_label=to_label, - ), - ) - _print_section( - console=console, - title="Changed Columns", - content=_render_changed_columns( - model_result=model_result, - verbose=verbose, - max_column_examples=max_column_examples, - example_render_options=example_render_options, - ), - ) - if verbose and model_result.unequal_row_samples: - example_content: RenderableType | None = _render_examples( - model_result=model_result, - max_column_examples=max_column_examples, - example_render_options=example_render_options, - ) - if example_content is not None: - _print_section( - console=console, - title="Examples", - content=example_content, - ) - if model_result.left_only_key_samples: - _print_section( - console=console, - title=f"{from_label} only", - content=_render_side_only_samples( - side_label=from_label, - key_samples=model_result.left_only_key_samples, - max_row_only_examples=max_row_only_examples, - ), - ) - if model_result.right_only_key_samples: - _print_section( - console=console, - title=f"{to_label} only", - content=_render_side_only_samples( - side_label=to_label, - key_samples=model_result.right_only_key_samples, - max_row_only_examples=max_row_only_examples, - ), - ) - - -def _render_overview(*, model_result: ModelDiffResult, mode_label: str) -> RenderableType: - table: Table = Table.grid(padding=(0, 2)) - table.add_column(style=_RICH_OBJECT_STYLE, justify="right") - table.add_column() - table.add_row( - "Input" if model_result.input_kind == DIFF_INPUT_KIND_QUERY else "Model", - model_result.name, - ) - table.add_row("Key", _primary_key_label(model_result)) - table.add_row("Comparison", _mode_display(model_result=model_result, mode_label=mode_label)) - if model_result.bounded_fallback: - table.add_row("Fallback", "no cursor configured; used full row diff") - if model_result.excluded_columns: - table.add_row("Excluded", ", ".join(model_result.excluded_columns)) - if model_result.row_result is not None: - table.add_row( - "Scope", - _row_scope_label(rows=model_result.row_result, unkeyed=model_result.unkeyed), - ) - tolerance_label: str | None = _tolerance_label(model_result.row_result) - if tolerance_label is not None: - table.add_row("Tolerances", tolerance_label) - return table - - -def _primary_key_label(model_result: ModelDiffResult) -> str: - if not model_result.unique_key: - return "" - return ", ".join(model_result.unique_key) - - -def _mode_display(*, model_result: ModelDiffResult, mode_label: str) -> str: - if model_result.bounded_fallback: - return f"{mode_label} (fallback to full row diff)" - return mode_label - - -def _tolerance_label(row_result: RowDiffResult | None) -> str | None: - if row_result is None: - return None - labels: list[str] = [] - column: RowDiffColumnResult - for column in row_result.column_results: - if column.tolerance is None: - continue - parts: list[str] = [] - if column.tolerance.absolute is not None: - parts.append(f"absolute={column.tolerance.absolute}") - if column.tolerance.relative is not None: - parts.append(f"relative={column.tolerance.relative}") - labels.append(f"{column.name} {' '.join(parts)}") - if not labels: - return None - return ", ".join(labels) - - -def _row_scope_label(*, rows: RowDiffResult, unkeyed: bool = False) -> str: - if unkeyed: - return f"exhaustive ({rows.population_count:,} multiset rows)" - if rows.sampling is None: - return f"exhaustive ({rows.population_count:,} union keys)" - if rows.is_exhaustive: - return ( - f"exhaustive ({rows.compared_count:,} union keys; " - f"configured sample limit {rows.sampling.row_limit:,})" - ) - percentage: str = _format_percentage( - numerator=rows.compared_count, - denominator=rows.population_count, - ) - return ( - f"sampled {rows.compared_count:,} of {rows.population_count:,} union keys " - f"({percentage}; seed {rows.sampling.seed})" - ) - - -def _render_coverage(*, rows: RowDiffResult, from_label: str, to_label: str) -> RenderableType: - if rows.left_coverage is None or rows.right_coverage is None: - return Text("Coverage was not collected.", style="italic") - table: Table = Table.grid(padding=(0, 2)) - table.add_column(style=_RICH_OBJECT_STYLE) - table.add_column(justify="right") - table.add_column() - table.add_row("Side", "Rows", "Cursor extent") - table.add_row( - from_label, - f"{rows.left_coverage.row_count:,}", - _cursor_extent(rows=rows, coverage=rows.left_coverage), - ) - table.add_row( - to_label, - f"{rows.right_coverage.row_count:,}", - _cursor_extent(rows=rows, coverage=rows.right_coverage), - ) - renderables: list[RenderableType] = [table] - if _cursor_coverage_differs(rows=rows): - renderables.extend( - ( - "", - Text( - "Warning: cursor coverage differs between sides; comparison continued " - "without narrowing the requested bounds.", - style="yellow", - ), - ) - ) - return Group(*renderables) - - -def _cursor_extent(*, rows: RowDiffResult, coverage: RowDiffCoverage) -> str: - if rows.cursor_column is None: - return "" - minimum: str = ( - str(coverage.minimum_cursor) if coverage.minimum_cursor is not None else "" - ) - maximum: str = ( - str(coverage.maximum_cursor) if coverage.maximum_cursor is not None else "" - ) - requested: str = _requested_cursor_bounds(rows=rows) - return f"{minimum} to {maximum} (requested {requested})" - - -def _requested_cursor_bounds(*, rows: RowDiffResult) -> str: - start: str = str(rows.cursor_start) if rows.cursor_start is not None else "unbounded" - end: str = str(rows.cursor_end) if rows.cursor_end is not None else "unbounded" - return f"{start} to {end}" - - -def _cursor_coverage_differs(*, rows: RowDiffResult) -> bool: - if rows.cursor_column is None or rows.left_coverage is None or rows.right_coverage is None: - return False - return ( - rows.left_coverage.minimum_cursor != rows.right_coverage.minimum_cursor - or rows.left_coverage.maximum_cursor != rows.right_coverage.maximum_cursor - ) - - -def _render_schema_summary(model_result: ModelDiffResult) -> RenderableType: - schema_result: SchemaDiffResult = model_result.schema_result - schema_diff_count: int = ( - len(schema_result.added_columns) - + len(schema_result.removed_columns) - + len(schema_result.type_changed_columns) - ) - if ( - not schema_result.added_columns - and not schema_result.removed_columns - and not schema_result.type_changed_columns - ): - return Text("No schema differences.", style="italic") - lines: list[str] = [f"schema differences: {schema_diff_count:,}"] - if schema_result.added_columns: - lines.append(f"added columns: {len(schema_result.added_columns):,}") - if schema_result.removed_columns: - lines.append(f"removed columns: {len(schema_result.removed_columns):,}") - if schema_result.type_changed_columns: - lines.append(f"type changes: {len(schema_result.type_changed_columns):,}") - return Group(*lines) - - -def _print_section(*, console: Console, title: str, content: RenderableType) -> None: - console.print(Text(title, style=_RICH_SECTION_STYLE)) - console.print(Text("▔" * len(title), style=_RICH_SECTION_STYLE)) - console.print(content) - console.print() - - -def _render_rows(*, rows: RowDiffResult, from_label: str, to_label: str) -> RenderableType: - counts: Table = Table.grid(padding=(0, 2)) - counts.add_column(justify="center") - counts.add_column(justify="center") - counts.add_column(justify="center") - counts.add_row(f"{from_label} count", "", f"{to_label} count") - counts.add_row( - f"{rows.left_count:,}", - _render_delta_label(left_count=rows.left_count, right_count=rows.right_count), - f"{rows.right_count:,}", - ) - - joined_count: int = rows.equal_count + rows.unequal_count - renderables: list[RenderableType] = [] - left_box: Table = Table(show_header=False, padding=0, box=box.HEAVY_EDGE) - right_box: Table = Table(show_header=False, padding=0, box=box.HEAVY_EDGE) - for _ in range(5): - left_box.add_column() - right_box.add_column() - - if rows.left_only_count > 0: - left_box.add_row(*([Text("-", style=_RICH_REMOVED_STYLE)] * 5)) - left_box.add_section() - if rows.equal_count > 0: - left_box.add_row(*([" "] * 5)) - left_box.add_section() - right_box.add_row(*([" "] * 5)) - right_box.add_section() - if rows.unequal_count > 0: - left_box.add_row(*([" "] * 5)) - right_box.add_row(*([" "] * 5)) - right_box.add_section() - if rows.right_only_count > 0: - right_box.add_row(*([Text("+", style=_RICH_ADDED_STYLE)] * 5)) - - renderables.append(left_box) - renderables.append(_render_row_separator(rows)) - renderables.append(Group("\n", right_box) if rows.left_only_count > 0 else right_box) - renderables.append( - _render_row_stats( - rows=rows, joined_count=joined_count, from_label=from_label, to_label=to_label - ) - ) - renderables.append(_render_joined_annotation(rows=rows, joined_count=joined_count)) - - return Group(counts, "", RichColumns(renderables, padding=0), "", f"joined: {joined_count:,}") - - -def _render_delta_label(*, left_count: int, right_count: int) -> str: - if left_count == 0: - return "(no baseline)" - if left_count == right_count: - return "(no change)" - if right_count > left_count: - return f"(+{((right_count / left_count) - 1):.2%})" - return f"(-{(1 - (right_count / left_count)):.2%})" - - -def _render_row_separator(rows: RowDiffResult) -> RenderableType: - joined: list[RenderableType] = [] - if rows.left_only_count > 0: - joined.append("\n") - if rows.equal_count > 0: - joined.append("╌" * 3) - joined.append(Text(" = ", style=_RICH_SECTION_STYLE)) - if rows.unequal_count > 0: - joined.append("╌" * 3) - joined.append(Text(" ≠ ", style=_RICH_SECTION_STYLE)) - joined.append("╌" * 3) - return Group(*joined) - - -def _render_row_stats( - *, - rows: RowDiffResult, - joined_count: int, - from_label: str, - to_label: str, -) -> RenderableType: - table: Table = Table(show_header=False, box=box.ROUNDED, padding=(0, 0, 0, 1)) - table.add_column(justify="right") - table.add_column(justify="left") - table.add_column(justify="right") - table.add_row( - f"{rows.left_only_count:,}", - f"{from_label} only", - f"({_format_percentage(numerator=rows.left_only_count, denominator=rows.left_count)})", - ) - table.add_section() - table.add_row( - f"{rows.equal_count:,}", - "equal", - f"({_format_percentage(numerator=rows.equal_count, denominator=joined_count)})", - ) - table.add_section() - table.add_row( - f"{rows.unequal_count:,}", - "unequal", - f"({_format_percentage(numerator=rows.unequal_count, denominator=joined_count)})", - ) - table.add_section() - table.add_row( - f"{rows.right_only_count:,}", - f"{to_label} only", - f"({_format_percentage(numerator=rows.right_only_count, denominator=rows.right_count)})", - ) - return table - - -def _render_joined_annotation(*, rows: RowDiffResult, joined_count: int) -> RenderableType: - if joined_count == 0: - return "" - lines: list[RenderableType] = [] - if rows.left_only_count > 0: - lines.append("\n") - lines.append("╌╮") - lines.append(" │") - if rows.equal_count > 0 and rows.unequal_count > 0: - lines.append(f"╌├╴ {joined_count:,} joined") - lines.append(" │") - lines.append("╌╯") - return Group(*lines) - - -def _render_changed_columns( - *, - model_result: ModelDiffResult, - verbose: bool, - max_column_examples: int, - example_render_options: DiffExampleRenderOptions, -) -> RenderableType: - row_result: RowDiffResult | None = model_result.row_result - if row_result is None: - return Text("Schema-only diff; row comparison skipped.", style="italic") - mismatched_columns: tuple[RowDiffColumnResult, ...] = tuple( - sorted( - (column for column in row_result.column_results if column.mismatched_count > 0), - key=lambda column: (-column.mismatched_count, column.name), - ) - ) - if not mismatched_columns: - message: str = ( - "No changed columns." - if row_result.is_exhaustive - else "No changed columns in sampled keys." - ) - return Text(message, style="italic") - table: Table = Table(show_header=False) - table.add_column(style=_RICH_OBJECT_STYLE, max_width=32, overflow="fold") - table.add_column(justify="right") - table.add_column(justify="right") - visible_columns: tuple[RowDiffColumnResult, ...] = mismatched_columns[:_TOP_CHANGED_COLUMNS] - column: RowDiffColumnResult - for column in visible_columns: - match_rate: str = _format_percentage( - numerator=_matching_rows(row_result=row_result, column=column), - denominator=row_result.joined_count, - ) - table.add_row( - column.name, - f"mismatches={column.mismatched_count:,}", - f"match={match_rate}", - ) - if len(mismatched_columns) > len(visible_columns): - table.add_section() - table.add_row( - Text("...", style=_RICH_MUTED_STYLE), - Text( - f"and {len(mismatched_columns) - len(visible_columns):,} more", - style=_RICH_MUTED_STYLE, - ), - Text("", style=_RICH_MUTED_STYLE), - ) - if not verbose: - example_content: RenderableType | None = _render_examples( - model_result=model_result, - max_column_examples=max_column_examples, - visible_column_names=tuple(column.name for column in visible_columns), - example_render_options=example_render_options, - ) - if example_content is not None: - return Group( - table, - "", - example_content, - "", - Text("Use --verbose to show more example row changes.", style=_RICH_MUTED_STYLE), - ) - return table - return table - - -def _matching_rows(*, row_result: RowDiffResult, column: RowDiffColumnResult) -> int: - return max(row_result.joined_count - column.mismatched_count, 0) - - -def _format_percentage(*, numerator: int | float, denominator: int | float) -> str: - if denominator == 0: - return "0.00%" - return f"{(numerator / denominator):.2%}" - - -def _render_examples( - *, - model_result: ModelDiffResult, - max_column_examples: int, - visible_column_names: tuple[str, ...] | None = None, - example_render_options: DiffExampleRenderOptions, -) -> RenderableType | None: - blocks: list[RenderableType] = [] - grouped_examples: dict[str, list[str]] = {} - sample_row: RowDiffSampleRow - for sample_row in model_result.unequal_row_samples: - key_label: str = ", ".join( - f"{key}={render_key_value(value)}" for key, value in sample_row.key_values - ) - cell: RowDiffSampleCell - for cell in sample_row.changed_cells: - left_value, right_value = render_example_pair( - left_value=cell.left_value, - right_value=cell.right_value, - options=example_render_options, - ) - difference_label: str = ( - f" (first difference at character {left_value.first_difference + 1:,})" - if (left_value.truncated or right_value.truncated) - and left_value.first_difference is not None - else "" - ) - grouped_examples.setdefault(cell.name, []).append( - f"{key_label} | {display_example_value(left_value)} -> " - f"{display_example_value(right_value)}{difference_label}" - ) - column_name: str - for column_name in sorted(grouped_examples): - if visible_column_names is not None and column_name not in visible_column_names: - continue - blocks.append(Text(column_name, style=_RICH_OBJECT_STYLE)) - all_examples: list[str] = grouped_examples[column_name] - visible_examples: list[str] = all_examples[:max_column_examples] - example: str - for example in visible_examples: - blocks.append(Text(f" - {example}")) - if len(all_examples) > len(visible_examples): - blocks.append( - Text( - f" showing {len(visible_examples):,} of {len(all_examples):,} examples", - style=_RICH_MUTED_STYLE, - ) - ) - blocks.append("") - return Group(*blocks[:-1]) if blocks else None - - -def _render_side_only_samples( - *, - side_label: str, - key_samples: tuple[tuple[tuple[str, object], ...], ...], - max_row_only_examples: int, -) -> RenderableType: - blocks: list[RenderableType] = [] - visible_samples: tuple[tuple[tuple[str, object], ...], ...] = key_samples[ - :max_row_only_examples - ] - sample: tuple[tuple[str, object], ...] - for sample in visible_samples: - blocks.append( - Text(" - " + " | ".join(f"{key}={render_key_value(value)}" for key, value in sample)) - ) - if len(key_samples) > len(visible_samples): - truncation_message: str = ( - f" showing {len(visible_samples):,} of {len(key_samples):,} {side_label} only rows" - ) - blocks.append( - Text( - truncation_message, - style=_RICH_MUTED_STYLE, - ) - ) - return Group(*blocks) if blocks else Text("No side-only samples collected.", style="italic") diff --git a/src/sqlbuild/cli/commands/_helpers/diff/outputs.py b/src/sqlbuild/cli/commands/_helpers/diff/outputs.py deleted file mode 100644 index 05e8c73ccd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/outputs.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Diff command output writing and exit-code phases.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.diff.json_output import ( - render_diff_json_output, - write_diff_json_output, -) -from sqlbuild.cli.commands._helpers.diff.output import has_diff_failures, render_diff_output -from sqlbuild.cli.commands.constants import DEFAULT_DIFF_MAX_VALUE_LENGTH -from sqlbuild.cli.commands.models import ( - DiffCommandRequest, - DiffExampleRenderOptions, - DirectDiffPreparation, -) -from sqlbuild.executor.diff.models import DiffExecutionResult -from sqlbuild.presentation.main.supports_color import supports_color - - -def write_direct_diff_output( - *, - request: DiffCommandRequest, - preparation: DirectDiffPreparation, - result: DiffExecutionResult, - phase_seconds: dict[str, float] | None = None, - outcome: str | None = None, -) -> None: - """Write direct diff output.""" - - example_render_options: DiffExampleRenderOptions = _example_render_options(request=request) - write_diff_json_output( - path=request.json_output_path, - result=result, - from_label=preparation.from_target, - to_label=preparation.to_target, - example_render_options=example_render_options, - phase_seconds=phase_seconds, - outcome=outcome, - ) - if request.json_output: - print( - render_diff_json_output( - result=result, - from_label=preparation.from_target, - to_label=preparation.to_target, - example_render_options=example_render_options, - phase_seconds=phase_seconds, - outcome=outcome, - ) - ) - else: - print( - render_diff_output( - result=result, - from_label=preparation.from_target, - to_label=preparation.to_target, - mode_label=_mode_label(request=request), - use_color=not request.no_color and supports_color(), - verbose=request.verbose, - max_column_examples=preparation.effective_max_column_examples, - max_row_only_examples=preparation.effective_max_row_only_examples, - example_render_options=example_render_options, - outcome=outcome, - ) - ) - - -def resolve_diff_exit_code(result: DiffExecutionResult) -> int: - """Resolve the diff exit code from failure state.""" - - return 1 if has_diff_failures(result) else 0 - - -def _example_render_options(*, request: DiffCommandRequest) -> DiffExampleRenderOptions: - max_value_length: int | None = ( - None - if request.full_example_values - else request.max_value_length or DEFAULT_DIFF_MAX_VALUE_LENGTH - ) - return DiffExampleRenderOptions( - max_value_length=max_value_length, - suppress_values=request.suppress_example_values, - ) - - -def _mode_label(*, request: DiffCommandRequest) -> str: - if request.schema_only: - return "schema-only" - if request.bounded: - return f"bounded {request.bounded}" - return "full" diff --git a/src/sqlbuild/cli/commands/_helpers/diff/size_guard.py b/src/sqlbuild/cli/commands/_helpers/diff/size_guard.py deleted file mode 100644 index bfb88e35fc..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/size_guard.py +++ /dev/null @@ -1,193 +0,0 @@ -"""Default full model diff size limits and guard error rendering.""" - -from __future__ import annotations - -import json -import shlex - -from sqlbuild.cli.commands.exceptions import DiffSizeGuardError -from sqlbuild.cli.commands.models import DiffCommandRequest -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.diff.exceptions import FullDiffSizeGuardError -from sqlbuild.executor.diff.models import FullDiffModelSize, FullDiffSideSize, FullDiffSizeLimits -from sqlbuild.spec.contracts.constants import DEFAULT_DIFF_MAX_FULL_ROWS -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config - -_BOUNDED_WINDOW_PLACEHOLDER: str = "" - - -def resolve_full_diff_size_limits( - *, - request: DiffCommandRequest, - discovered_inputs: DiscoveredProjectInputs, - from_target: str, - to_target: str, -) -> FullDiffSizeLimits | None: - """Return per-side row limits when no explicit comparison mode was requested.""" - - if request.full or request.schema_only or request.bounded is not None: - return None - return FullDiffSizeLimits( - left_target=from_target, - right_target=to_target, - left_max_rows=_target_max_full_rows( - discovered_inputs=discovered_inputs, target_name=from_target - ), - right_max_rows=_target_max_full_rows( - discovered_inputs=discovered_inputs, target_name=to_target - ), - ) - - -def build_diff_size_guard_error( - *, - request: DiffCommandRequest, - from_target: str, - to_target: str, - error: FullDiffSizeGuardError, -) -> DiffSizeGuardError: - """Describe every blocked model and the explicit commands that replace the guarded run.""" - - commands: dict[str, str | None] = _replacement_commands( - request=request, - from_target=from_target, - to_target=to_target, - blocked=error.blocked, - ) - lines: list[str] = [error.message] - model: FullDiffModelSize - for model in error.blocked: - lines.append( - f" {model.name}: {_side_text(model.left)}; {_side_text(model.right)}" - + ("" if model.has_cursor else "; no cursor") - ) - lines.append("Run one of these instead:") - lines.append(f" {commands['full']}") - if commands["bounded"] is not None: - lines.append(f" {commands['bounded']}") - lines.append(f" {commands['schema_only']}") - over_targets: tuple[str, ...] = _over_limit_targets(blocked=error.blocked) - bounded_help: str = ( - "" - if commands["bounded"] is None - else ", --bounded compares a recent cursor window " - f"(replace {_BOUNDED_WINDOW_PLACEHOLDER} with a duration such as 7d)" - ) - return DiffSizeGuardError( - "\n".join(lines), - code="C270", - help=( - f"--full compares every row{bounded_help}, and --schema-only compares columns " - "only; or raise " - + " and ".join(f"targets.{target}.diff.max_full_rows" for target in over_targets) - ), - details={ - "from_target": from_target, - "to_target": to_target, - "models": [_model_payload(model) for model in error.blocked], - "commands": commands, - }, - ) - - -def _over_limit_targets(*, blocked: tuple[FullDiffModelSize, ...]) -> tuple[str, ...]: - targets: dict[str, None] = {} - model: FullDiffModelSize - for model in blocked: - side: FullDiffSideSize - for side in (model.left, model.right): - if side.exceeds_limit: - targets[side.target] = None - return tuple(targets) - - -def _target_max_full_rows( - *, discovered_inputs: DiscoveredProjectInputs, target_name: str -) -> int | None: - configured: int | str | None = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ).diff.max_full_rows - if configured is None: - return DEFAULT_DIFF_MAX_FULL_ROWS - return configured if isinstance(configured, int) else None - - -def _side_text(side: FullDiffSideSize) -> str: - limit: str = "unlimited" if side.max_rows is None else f"{side.max_rows:,}" - if side.row_count is None: - size: str = "size unknown" if side.detail is None else f"size unknown: {side.detail}" - else: - size = f"{side.row_count:,} rows" - return f"{side.target} {size}, limit {limit}" - - -def _model_payload(model: FullDiffModelSize) -> dict[str, object]: - return { - "name": model.name, - "has_cursor": model.has_cursor, - "from": _side_payload(model.left), - "to": _side_payload(model.right), - } - - -def _side_payload(side: FullDiffSideSize) -> dict[str, object]: - return { - "target": side.target, - "relation": side.relation, - "row_count": side.row_count, - "max_full_rows": side.max_rows, - "exceeds_limit": side.exceeds_limit, - "detail": side.detail, - } - - -def _replacement_commands( - *, - request: DiffCommandRequest, - from_target: str, - to_target: str, - blocked: tuple[FullDiffModelSize, ...], -) -> dict[str, str | None]: - scope: tuple[str, ...] = _scope_arguments(request=request) - project: tuple[str, ...] = ( - () if request.project_dir is None else ("--project-dir", str(request.project_dir)) - ) - prefix: tuple[str, ...] = ("sqb", *project, "diff", f"{from_target}:{to_target}") - bounded_available: bool = all(model.has_cursor for model in blocked) - return { - "full": _join((*prefix, "--full", *scope)), - "bounded": ( - _join((*prefix, "--bounded", _BOUNDED_WINDOW_PLACEHOLDER, *scope)) - if bounded_available - else None - ), - "schema_only": _join((*prefix, "--schema-only", *scope)), - } - - -def _scope_arguments(*, request: DiffCommandRequest) -> tuple[str, ...]: - arguments: list[str] = [] - if request.select: - arguments.extend(("--select", *request.select)) - if request.exclude: - arguments.extend(("--exclude", *request.exclude)) - if request.unique_key_override: - arguments.extend(("--key", *request.unique_key_override)) - if request.unkeyed: - arguments.append("--unkeyed") - if request.excluded_columns_override: - arguments.extend(("--exclude-column", *request.excluded_columns_override)) - tolerance: str - for tolerance in request.tolerance_overrides: - arguments.extend(("--tolerance", tolerance)) - if request.cli_vars: - arguments.extend(("--vars", json.dumps(request.cli_vars, sort_keys=True))) - return tuple(arguments) - - -def _join(parts: tuple[str, ...]) -> str: - return " ".join( - part if part == _BOUNDED_WINDOW_PLACEHOLDER else shlex.quote(part) for part in parts - ) diff --git a/src/sqlbuild/cli/commands/_helpers/diff/validation.py b/src/sqlbuild/cli/commands/_helpers/diff/validation.py deleted file mode 100644 index 8320f2f72f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/validation.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Validation helpers for diff command arguments.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.exceptions import CliUserError - - -def parse_diff_name_range(name_range: str | None) -> tuple[str, str | None]: - """Parse a diff name range in FROM:TO or FROM form; a bare FROM leaves TO unset.""" - - if name_range is None: - raise CliUserError("diff requires FROM:TO", code="C208") - from_name: str - separator: str - to_name: str - from_name, separator, to_name = name_range.partition(":") - if not from_name or (separator and (not to_name or separator in to_name)): - raise CliUserError("diff range must be FROM:TO or FROM", code="C209") - return from_name, to_name or None diff --git a/src/sqlbuild/cli/commands/_helpers/entry/observability.py b/src/sqlbuild/cli/commands/_helpers/entry/observability.py index 0904aac803..ccfc2f01f9 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/observability.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/observability.py @@ -13,15 +13,13 @@ from sqlbuild._native import ( NativeProgressProjector, TerminalEventIndex, + machine_output_mode, terminal_event_index_scope, ) from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.constants import ( - CSV_OUTPUT_FORMAT, DEBUG_COMMAND, - JSON_OUTPUT_FORMAT, LINEAGE_COMMAND, - QUERY_COMMAND, ) from sqlbuild.cli.commands.main.entrypoint._dispatch_with_output_capture import ( configured_output_capture_scope, @@ -78,14 +76,7 @@ def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> Iterato subscriber=terminal_index.consume, accepts_opaque=False, ) - machine_output: bool = ( - args.json - or (args.command == LINEAGE_COMMAND and args.lineage_format == JSON_OUTPUT_FORMAT) - or ( - args.command == QUERY_COMMAND - and args.query_format in {CSV_OUTPUT_FORMAT, JSON_OUTPUT_FORMAT} - ) - ) + machine_output: bool = machine_output_mode(args) projector: NativeProgressProjector = NativeProgressProjector( stream=sys.stderr if machine_output or args.debug or args.command == DEBUG_COMMAND diff --git a/src/sqlbuild/cli/commands/_helpers/load/execution.py b/src/sqlbuild/cli/commands/_helpers/load/execution.py deleted file mode 100644 index c3617fe427..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/execution.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Load command execution phase.""" - -from __future__ import annotations - -import time - -from sqlbuild._native import ConnectionProgressReporter, ExecutionEventWriter -from sqlbuild.cli.commands._helpers.input.parsing import ( - parse_cursor_integer, - parse_cursor_timestamp, -) -from sqlbuild.cli.commands.classes.load_progress_reporter import LoadProgressReporter -from sqlbuild.cli.commands.models import ( - LoadCommandRequest, - LoadExecutionPreparation, - LoadInvocation, - LoadRunOutcome, -) -from sqlbuild.executor.load.main.run import run_load_pipeline -from sqlbuild.executor.load.models import ( - LoadCallbacks, - LoadExecutionResult, - LoadRuntimeParams, -) -from sqlbuild.executor.scheduling.types import ExecutionStatus - - -def execute_load_plan( - *, - request: LoadCommandRequest, - invocation: LoadInvocation, - preparation: LoadExecutionPreparation, -) -> LoadRunOutcome: - """Execute selected source loads.""" - - start: float = time.monotonic() - event_writer: ExecutionEventWriter = ExecutionEventWriter() - load_progress: LoadProgressReporter = LoadProgressReporter( - stream=invocation.progress_stream, - use_color=invocation.use_color, - source_order={ - source.name: index for index, source in enumerate(invocation.selected_sources, start=1) - }, - total_count=len(invocation.selected_sources), - ) - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=preparation.adapter_name, - stream=invocation.progress_stream, - blank_line_after_complete=True, - use_color=invocation.use_color, - ) - - def on_load_complete(result: LoadExecutionResult) -> None: - load_progress.on_complete(result) - event_writer.write_build_result(result=result, plan=None, command="load") - - try: - results: tuple[LoadExecutionResult, ...] = run_load_pipeline( - sources=invocation.selected_sources, - reference_sources=invocation.reference_sources, - loader_functions=invocation.discovered_inputs.loader_functions, - connection_config=preparation.connection_config, - adapter=preparation.adapter, - max_concurrency=preparation.effective_concurrency, - runtime=LoadRuntimeParams( - run_id=preparation.run_id, - runtime_dir=invocation.effective_project_dir / "target", - target=preparation.target_name, - vars=preparation.effective_vars, - is_reload=request.reload, - start_cursor_ts=parse_cursor_timestamp( - preparation.effective_cursor_overrides.start_ts - ), - end_cursor_ts=parse_cursor_timestamp(preparation.effective_cursor_overrides.end_ts), - start_cursor_int=parse_cursor_integer( - preparation.effective_cursor_overrides.start_int - ), - end_cursor_int=parse_cursor_integer(preparation.effective_cursor_overrides.end_int), - use_color=invocation.use_color, - providers=preparation.provider_session.providers, - project_relations=preparation.project_relations, - ), - callbacks=LoadCallbacks( - on_load_start=load_progress.on_start, - on_load_progress=lambda source, message: load_progress.on_progress( - source=source, message=message - ), - on_load_complete=on_load_complete, - on_connection_start=connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - ) - finally: - event_writer.close() - return LoadRunOutcome( - results=results, - elapsed=time.monotonic() - start, - success_count=sum( - 1 - for result in results - if result.status == ExecutionStatus.SUCCESS and not result.warning_messages - ), - fail_count=sum(1 for result in results if result.status == ExecutionStatus.FAILED), - skip_count=sum(1 for result in results if result.status == ExecutionStatus.SKIPPED), - warn_count=sum( - 1 - for result in results - if result.status == ExecutionStatus.SUCCESS and result.warning_messages - ), - ) diff --git a/src/sqlbuild/cli/commands/_helpers/load/invocation.py b/src/sqlbuild/cli/commands/_helpers/load/invocation.py deleted file mode 100644 index 0a522f3140..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/invocation.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Load command invocation resolution phase.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import discover_project_inputs -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.cli.commands._helpers.load.selection import ( - select_load_entries, - select_load_reference_entries, - select_load_relation_entries, -) -from sqlbuild.cli.commands.models import LoadCommandRequest, LoadInvocation -from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config -from sqlbuild.compiler.compile.main.effective_target import build_effective_target_config -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import SourceEntry, TargetConfig - - -def resolve_load_invocation(*, request: LoadCommandRequest) -> LoadInvocation: - """Resolve discovery, source selection, and output context for load.""" - - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - reject_unit_test_selectors( - select=request.select, exclude=request.exclude, discovered_inputs=discovered_inputs - ) - target_config: TargetConfig | None = build_effective_target_config( - discovered_inputs=discovered_inputs, - selected_target=request.selected_target, - ) - loader_default_database, loader_default_schema = _effective_loader_defaults( - discovered_inputs=discovered_inputs, - selected_target=request.selected_target, - target_config=target_config, - cli_vars=request.cli_vars, - ) - selected_sources: tuple[SourceEntry, ...] = select_load_entries( - discovered_inputs=discovered_inputs, - select=request.select, - exclude=request.exclude, - target_config=target_config, - loader_default_database=loader_default_database, - loader_default_schema=loader_default_schema, - ) - reference_sources: tuple[SourceEntry, ...] = select_load_reference_entries( - discovered_inputs=discovered_inputs, - selected_sources=selected_sources, - target_config=target_config, - loader_default_database=loader_default_database, - loader_default_schema=loader_default_schema, - ) - machine_output: bool = request.json_output - use_color: bool = not request.no_color and not machine_output and supports_color() - progress_stream: TextIO = sys.stderr if machine_output else sys.stdout - return LoadInvocation( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - selected_sources=selected_sources, - reference_sources=reference_sources, - relation_sources=select_load_relation_entries( - discovered_inputs=discovered_inputs, target_config=target_config - ), - use_color=use_color, - progress_stream=progress_stream, - ) - - -def _effective_loader_defaults( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - target_config: TargetConfig | None, - cli_vars: dict[str, object] | None, -) -> tuple[str | None, str | None]: - connection: dict[str, object] = build_effective_connection_config( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - cli_vars=cli_vars, - ) - connection_database: str | None = _non_empty_string(connection.get("database")) - if ( - resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - == BuiltinAdapter.DUCKDB - ): - connection_database = None - return ( - (target_config.database if target_config is not None else None) or connection_database, - (target_config.schema if target_config is not None else None) - or _non_empty_string(connection.get("schema")), - ) - - -def _non_empty_string(value: object) -> str | None: - return value.strip() if isinstance(value, str) and value.strip() else None diff --git a/src/sqlbuild/cli/commands/_helpers/load/outputs.py b/src/sqlbuild/cli/commands/_helpers/load/outputs.py deleted file mode 100644 index 1f1f2e4247..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/outputs.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Load command output phases.""" - -from __future__ import annotations - -from sqlbuild._native import ( - format_completion_line, - format_load_execution_json, - format_summary_footer, - write_execution_header, - write_execution_json_output, -) -from sqlbuild.cli.commands.classes.load_progress_reporter import format_load_footer -from sqlbuild.cli.commands.models import ( - LoadCommandRequest, - LoadExecutionPreparation, - LoadInvocation, - LoadRunOutcome, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.surface_header import format_surface_header -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.types import CompletionState -from sqlbuild.spec.contracts.models import SourceEntry - - -def write_load_ready_output(*, invocation: LoadInvocation) -> None: - """Write selected source summary header.""" - - style: CliStyle = CliStyle(use_color=invocation.use_color) - ready_header: str = format_surface_header( - style=style, - title="Load ready", - context=f"{len(invocation.selected_sources)} selected", - ) - invocation.progress_stream.write(f"\n{ready_header}\n\n") - invocation.progress_stream.flush() - - -def write_empty_load_output(*, request: LoadCommandRequest, invocation: LoadInvocation) -> None: - """Write successful no-op load output and optional JSON.""" - - style: CliStyle = CliStyle(use_color=invocation.use_color) - invocation.progress_stream.write(style.muted(" No managed sources selected.") + "\n") - counts_summary: str = format_summary_footer( - counts=( - ("PASS", 0), - ("WARN", 0), - ("FAIL", 0), - ("SKIP", 0), - ("TOTAL", 0), - ), - use_color=invocation.use_color, - elapsed="0.00s", - ) - completion: str = format_completion_line( - style=style, - state=CompletionState.OK, - label="Completed successfully", - summary=counts_summary, - ) - invocation.progress_stream.write(f"\n{completion}\n") - invocation.progress_stream.flush() - write_execution_json_output( - payload=format_load_execution_json(results=()), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def write_load_plan_output(*, invocation: LoadInvocation) -> None: - """Write selected loader and source names before execution.""" - - style: CliStyle = CliStyle(use_color=invocation.use_color) - loader_entries: tuple[SourceEntry, ...] = tuple( - source for source in invocation.selected_sources if _is_loader_node(source) - ) - source_entries: tuple[SourceEntry, ...] = tuple( - source for source in invocation.selected_sources if not _is_loader_node(source) - ) - if loader_entries: - invocation.progress_stream.write(style.section(f"Loaders ({len(loader_entries)})")) - invocation.progress_stream.write("\n") - _write_source_names(invocation=invocation, sources=loader_entries) - invocation.progress_stream.write("\n") - invocation.progress_stream.write(style.section(f"Sources ({len(source_entries)})")) - invocation.progress_stream.write("\n") - _write_source_names(invocation=invocation, sources=source_entries) - invocation.progress_stream.write("\n") - invocation.progress_stream.flush() - - -def write_load_execution_header( - *, invocation: LoadInvocation, preparation: LoadExecutionPreparation -) -> None: - """Write direct execution header for load.""" - - write_execution_header( - stream=invocation.progress_stream, - command="sqb load", - target=None, - concurrency=preparation.effective_concurrency, - use_color=invocation.use_color, - ) - - -def write_load_completion_output( - *, request: LoadCommandRequest, invocation: LoadInvocation, outcome: LoadRunOutcome -) -> None: - """Write final load footer and optional JSON.""" - - invocation.progress_stream.write( - format_load_footer( - results=outcome.results, - success_count=outcome.success_count, - warn_count=outcome.warn_count, - fail_count=outcome.fail_count, - skip_count=outcome.skip_count, - elapsed=outcome.elapsed, - use_color=invocation.use_color, - ) - ) - invocation.progress_stream.flush() - write_execution_json_output( - payload=format_load_execution_json(results=outcome.results), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def resolve_load_exit_code(outcome: LoadRunOutcome) -> int: - """Return the shell exit code for a load outcome.""" - - return 0 if outcome.fail_count == 0 else 1 - - -def _write_source_names(*, invocation: LoadInvocation, sources: tuple[SourceEntry, ...]) -> None: - style: CliStyle = CliStyle(use_color=invocation.use_color) - source: SourceEntry - source_index: int - for source_index, source in enumerate(sources): - connector: str = tree_connector(style=style, last=source_index == len(sources) - 1) - invocation.progress_stream.write(f"{connector} {source.name}\n") - - -def _is_loader_node(source: SourceEntry) -> bool: - return source.meta.get("sqlbuild_loader_node") is True diff --git a/src/sqlbuild/cli/commands/_helpers/load/planning.py b/src/sqlbuild/cli/commands/_helpers/load/planning.py deleted file mode 100644 index 98beea16dc..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/planning.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Load command execution preparation phase.""" - -from __future__ import annotations - -from sqlbuild._native import ( - render_source_relation, - resolve_project_connection_config, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.load.references import validate_reference_source_targets -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.models import ( - LoadCommandRequest, - LoadExecutionPreparation, - LoadInvocation, -) -from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config -from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config -from sqlbuild.compiler.planner.main.cursor_window.typed_cursor_overrides import ( - typed_cursor_overrides, -) -from sqlbuild.compiler.planner.models import CursorOverrides -from sqlbuild.provider.main.session import build_provider_session -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import SourceEntry - - -def prepare_load_execution( - *, request: LoadCommandRequest, invocation: LoadInvocation -) -> LoadExecutionPreparation: - """Prepare adapter, connection, runtime, concurrency, and providers for load.""" - - adapter_name: str = resolve_effective_adapter_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, - project_dir=invocation.effective_project_dir, - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=invocation.discovered_inputs, - project_dir=invocation.effective_project_dir, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - validate_reference_source_targets( - adapter=adapter, - connection_config=connection_config, - selected_sources=invocation.selected_sources, - reference_sources=invocation.reference_sources, - ) - target_name: str | None - effective_vars: dict[str, object] - run_id: str - target_name, effective_vars, run_id = build_effective_runtime_config( - discovered_inputs=invocation.discovered_inputs, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - effective_concurrency: int = max( - 1, - request.concurrency - if request.concurrency is not None - else build_effective_settings_config( - discovered_inputs=invocation.discovered_inputs - ).concurrency, - ) - return LoadExecutionPreparation( - adapter_name=adapter_name, - adapter=adapter, - connection_config=connection_config, - target_name=target_name, - effective_vars=effective_vars, - run_id=run_id, - effective_cursor_overrides=( - typed_cursor_overrides(cursor_overrides=request.cursor_overrides) or CursorOverrides() - ), - effective_concurrency=effective_concurrency, - provider_session=build_provider_session( - discovered_providers=invocation.discovered_inputs.providers - ), - project_relations=_load_project_relations(adapter=adapter, invocation=invocation), - ) - - -def _load_project_relations( - *, adapter: BaseAdapter, invocation: LoadInvocation -) -> dict[SqlResourceRef, str] | None: - """Return the source relations loaders must not hard-code, or None when enforcement is off.""" - - if not invocation.discovered_inputs.project_config.references.enforce_explicit: - return None - sources: dict[str, SourceEntry] = { - source.name: source - for source in ( - *invocation.relation_sources, - *invocation.reference_sources, - *invocation.selected_sources, - ) - if source.expression is None - } - return { - SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=name): render_source_relation( - entry=source, adapter=adapter - ) - for name, source in sources.items() - } diff --git a/src/sqlbuild/cli/commands/_helpers/load/references.py b/src/sqlbuild/cli/commands/_helpers/load/references.py deleted file mode 100644 index 067d27f96b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/references.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Helpers for source loader reference-only dependencies.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.spec.contracts.models import SourceEntry - - -def validate_reference_source_targets( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - selected_sources: tuple[SourceEntry, ...], - reference_sources: tuple[SourceEntry, ...], -) -> None: - """Validate skipped intermediate loader reference targets already exist.""" - - if not reference_sources: - return - selected_source_names: str = ", ".join(source.name for source in selected_sources) - connection: object = adapter.connect(connection_config) - try: - relation_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple( - ( - reference_source.database, - reference_source.schema, - reference_source.table or reference_source.name, - ) - for reference_source in reference_sources - ), - ) - reference_source: SourceEntry - for reference_source in reference_sources: - target_name: str = reference_source.table or reference_source.name - if relation_lookup.exists( - database=reference_source.database, - schema=reference_source.schema, - name=target_name, - ): - continue - raise CliUserError( - f"Selected source load requires intermediate loader '{reference_source.name}', " - f"but its target relation '{target_name}' does not exist; use " - f"+{selected_source_names} to refresh upstream ingress dependencies" - ) - finally: - adapter.close(connection) diff --git a/src/sqlbuild/cli/commands/_helpers/load/selection.py b/src/sqlbuild/cli/commands/_helpers/load/selection.py deleted file mode 100644 index 186d5ea33a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/load/selection.py +++ /dev/null @@ -1,632 +0,0 @@ -"""Selection helpers for sqb load source/loader graphs.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild._native import transitive_closure -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import LoadSelectionSets, LoadSelectorSets -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts -from sqlbuild.spec.contracts.models import LoaderDestinationParts, SourceEntry, TargetConfig - - -def select_load_entries( - *, - discovered_inputs: DiscoveredProjectInputs, - select: tuple[str, ...], - exclude: tuple[str, ...], - target_config: TargetConfig | None, - loader_default_database: str | None = None, - loader_default_schema: str | None = None, -) -> tuple[SourceEntry, ...]: - """Select source and intermediate loader execution entries for sqb load.""" - - discovered_sources: list[SourceEntry] = [] - for source_file in discovered_inputs.source_files: - discovered_sources.extend(source_file.source_entries) - sources: tuple[SourceEntry, ...] = _environment_sources( - sources=tuple(discovered_sources), target_config=target_config - ) - managed_sources: dict[str, SourceEntry] = { - source.name: source for source in sources if source.loader is not None - } - loaders: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in discovered_inputs.loader_functions - } - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in discovered_inputs.loader_functions - } - upstream_loaders: dict[str, tuple[str, ...]] = {} - for loader in discovered_inputs.loader_functions: - dependency_names: list[str] = [] - for dependency in loader.depends_on: - if dependency in loader_name_by_function: - dependency_names.append(loader_name_by_function[dependency]) - upstream_loaders[loader.name] = tuple(dependency_names) - source_by_loader: dict[str, tuple[str, ...]] = _source_names_by_loader(managed_sources) - - selected_sources: set[str] = set() - selected_loaders: set[str] = set() - directly_selected_loaders: set[str] = set() - raw_selectors: tuple[str, ...] = select or tuple(f"+{name}" for name in managed_sources) - selector_sets: LoadSelectorSets = _apply_selectors( - raw_selectors=raw_selectors, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - directly_selected_loaders=directly_selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - upstream_loaders=upstream_loaders, - ) - selected_sources = selector_sets.selected_sources - selected_loaders = selector_sets.selected_loaders - directly_selected_loaders = selector_sets.directly_selected_loaders - exclusion_sets, excluded_loaders = _apply_excludes( - exclude=exclude, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - upstream_loaders=upstream_loaders, - ) - selected_sources = exclusion_sets.selected_sources - selected_loaders = exclusion_sets.selected_loaders - - selected_loaders, selected_sources = _prune_missing_dependencies( - selected_loaders=selected_loaders, - selected_sources=selected_sources, - managed_sources=managed_sources, - upstream_loaders=upstream_loaders, - excluded_loaders=excluded_loaders, - ) - selected_loaders = _prune_unneeded_loaders( - selected_loaders=selected_loaders, - selected_sources=selected_sources, - directly_selected_loaders=directly_selected_loaders, - managed_sources=managed_sources, - upstream_loaders=upstream_loaders, - ) - - entries: list[SourceEntry] = [] - selected_terminal_loaders: set[str | None] = { - managed_sources[source_name].loader for source_name in selected_sources - } - loader_name: str - for loader_name in _topological_loader_order( - loader_names=selected_loaders, - upstream_loaders=upstream_loaders, - ): - if loader_name not in selected_terminal_loaders: - entries.append( - _loader_to_source_entry( - loader=loaders[loader_name], - default_database=loader_default_database, - default_schema=loader_default_schema, - ) - ) - entries.extend( - managed_sources[source_name] - for source_name in sources_order(sources=sources, selected_sources=selected_sources) - ) - return tuple(entries) - - -def select_load_reference_entries( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_sources: tuple[SourceEntry, ...], - target_config: TargetConfig | None, - loader_default_database: str | None = None, - loader_default_schema: str | None = None, -) -> tuple[SourceEntry, ...]: - """Return unselected upstream intermediate loader entries used only for refs.""" - - selected_names: frozenset[str] = frozenset(source.name for source in selected_sources) - loaders: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in discovered_inputs.loader_functions - } - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in discovered_inputs.loader_functions - } - upstream_loaders: dict[str, tuple[str, ...]] = {} - for loader in discovered_inputs.loader_functions: - dependency_names: list[str] = [] - for dependency in loader.depends_on: - if dependency in loader_name_by_function: - dependency_names.append(loader_name_by_function[dependency]) - upstream_loaders[loader.name] = tuple(dependency_names) - reference_loader_names: set[str] = set() - source: SourceEntry - for source in selected_sources: - if source.loader is None or source.loader not in upstream_loaders: - continue - dependency_name: str - for dependency_name in upstream_loaders[source.loader]: - reference_loader_names.update( - _upstream_loader_closure( - loader_name=dependency_name, upstream_loaders=upstream_loaders - ) - ) - return tuple( - _loader_to_source_entry( - loader=loaders[loader_name], - default_database=loader_default_database, - default_schema=loader_default_schema, - ) - for loader_name in _topological_loader_order( - loader_names=reference_loader_names, - upstream_loaders=upstream_loaders, - ) - if loader_name in loaders and loader_name not in selected_names - ) - - -def select_load_relation_entries( - *, discovered_inputs: DiscoveredProjectInputs, target_config: TargetConfig | None -) -> tuple[SourceEntry, ...]: - """Return every declared source resolved for the target, for hard-coded name warnings.""" - - discovered_sources: list[SourceEntry] = [] - for source_file in discovered_inputs.source_files: - discovered_sources.extend(source_file.source_entries) - return _environment_sources(sources=tuple(discovered_sources), target_config=target_config) - - -def sources_order( - *, sources: tuple[SourceEntry, ...], selected_sources: set[str] -) -> tuple[str, ...]: - return tuple(source.name for source in sources if source.name in selected_sources) - - -def _source_names_by_loader(managed_sources: dict[str, SourceEntry]) -> dict[str, tuple[str, ...]]: - grouped_source_names: dict[str, list[str]] = {} - source: SourceEntry - for source in managed_sources.values(): - if source.loader is None: - continue - grouped_source_names.setdefault(source.loader, []).append(source.name) - return { - loader_name: tuple(source_names) - for loader_name, source_names in grouped_source_names.items() - } - - -def _apply_selectors( - *, - raw_selectors: tuple[str, ...], - selected_sources: set[str], - selected_loaders: set[str], - directly_selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - upstream_loaders: dict[str, tuple[str, ...]], -) -> LoadSelectorSets: - selector: str - for selector in raw_selectors: - name, include_upstream, include_downstream = _parse_load_selector(selector) - _validate_load_selector(name=name, managed_sources=managed_sources, loaders=loaders) - selector_sets: LoadSelectorSets = _select_name( - name=name, - include_upstream=include_upstream, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - directly_selected_loaders=directly_selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - upstream_loaders=upstream_loaders, - ) - selected_sources = selector_sets.selected_sources - selected_loaders = selector_sets.selected_loaders - directly_selected_loaders = selector_sets.directly_selected_loaders - if include_downstream: - selection_sets: LoadSelectionSets = _select_downstream( - name=name, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - upstream_loaders=upstream_loaders, - ) - selected_sources = selection_sets.selected_sources - selected_loaders = selection_sets.selected_loaders - return LoadSelectorSets(selected_sources, selected_loaders, directly_selected_loaders) - - -def _apply_excludes( - *, - exclude: tuple[str, ...], - selected_sources: set[str], - selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - upstream_loaders: dict[str, tuple[str, ...]], -) -> tuple[LoadSelectionSets, set[str]]: - excluded_loaders: set[str] = set() - selector: str - for selector in exclude: - name, _, include_downstream = _parse_load_selector(selector) - _validate_load_selector(name=name, managed_sources=managed_sources, loaders=loaders) - selection_sets: LoadSelectionSets - selection_sets, excluded_loaders = _exclude_name( - name=name, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - excluded_loaders=excluded_loaders, - ) - selected_sources = selection_sets.selected_sources - selected_loaders = selection_sets.selected_loaders - if include_downstream: - selection_sets = _exclude_downstream( - name=name, - selected_sources=selected_sources, - selected_loaders=selected_loaders, - managed_sources=managed_sources, - loaders=loaders, - source_by_loader=source_by_loader, - upstream_loaders=upstream_loaders, - ) - selected_sources = selection_sets.selected_sources - selected_loaders = selection_sets.selected_loaders - return LoadSelectionSets(selected_sources, selected_loaders), excluded_loaders - - -def _select_name( - *, - name: str, - include_upstream: bool, - selected_sources: set[str], - selected_loaders: set[str], - directly_selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - upstream_loaders: dict[str, tuple[str, ...]], -) -> LoadSelectorSets: - if name in managed_sources: - selected_sources.add(name) - loader_name: str | None = managed_sources[name].loader - if loader_name is not None: - selected_loaders.add(loader_name) - if include_upstream: - selected_loaders.update( - _upstream_loader_closure( - loader_name=loader_name, upstream_loaders=upstream_loaders - ) - ) - return LoadSelectorSets(selected_sources, selected_loaders, directly_selected_loaders) - if name in source_by_loader: - selected_sources.update(source_by_loader[name]) - selected_loaders.add(name) - if include_upstream: - selected_loaders.update( - _upstream_loader_closure(loader_name=name, upstream_loaders=upstream_loaders) - ) - return LoadSelectorSets(selected_sources, selected_loaders, directly_selected_loaders) - if name in loaders: - selected_loaders.add(name) - directly_selected_loaders.add(name) - if include_upstream: - selected_loaders.update( - _upstream_loader_closure(loader_name=name, upstream_loaders=upstream_loaders) - ) - return LoadSelectorSets(selected_sources, selected_loaders, directly_selected_loaders) - - -def _select_downstream( - *, - name: str, - selected_sources: set[str], - selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - upstream_loaders: dict[str, tuple[str, ...]], -) -> LoadSelectionSets: - downstream_loaders, downstream_sources = _downstream_closure( - loader_name=_selected_loader_name( - name=name, - managed_sources=managed_sources, - loaders=loaders, - ), - upstream_loaders=upstream_loaders, - source_by_loader=source_by_loader, - ) - selected_loaders.update(downstream_loaders) - selected_sources.update(downstream_sources) - return LoadSelectionSets(selected_sources, selected_loaders) - - -def _exclude_name( - *, - name: str, - selected_sources: set[str], - selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - excluded_loaders: set[str], -) -> tuple[LoadSelectionSets, set[str]]: - if name in managed_sources: - selected_sources.discard(name) - loader_name: str | None = managed_sources[name].loader - if loader_name is not None and not any( - source_name in selected_sources for source_name in source_by_loader.get(loader_name, ()) - ): - selected_loaders.discard(loader_name) - return LoadSelectionSets(selected_sources, selected_loaders), excluded_loaders - if name in source_by_loader: - source_name: str - for source_name in source_by_loader[name]: - selected_sources.discard(source_name) - selected_loaders.discard(name) - excluded_loaders.add(name) - return LoadSelectionSets(selected_sources, selected_loaders), excluded_loaders - if name in loaders: - selected_loaders.discard(name) - excluded_loaders.add(name) - return LoadSelectionSets(selected_sources, selected_loaders), excluded_loaders - - -def _exclude_downstream( - *, - name: str, - selected_sources: set[str], - selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], - source_by_loader: dict[str, tuple[str, ...]], - upstream_loaders: dict[str, tuple[str, ...]], -) -> LoadSelectionSets: - downstream_loaders, downstream_sources = _downstream_closure( - loader_name=_selected_loader_name( - name=name, - managed_sources=managed_sources, - loaders=loaders, - ), - upstream_loaders=upstream_loaders, - source_by_loader=source_by_loader, - ) - selected_loaders.difference_update(downstream_loaders) - selected_sources.difference_update(downstream_sources) - return LoadSelectionSets(selected_sources, selected_loaders) - - -def _selected_loader_name( - *, - name: str, - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], -) -> str | None: - if name in loaders: - return name - return managed_sources[name].loader - - -def _environment_sources( - *, sources: tuple[SourceEntry, ...], target_config: TargetConfig | None -) -> tuple[SourceEntry, ...]: - return tuple( - replace( - source, - database=source.database - if source.database is not None or target_config is None - else target_config.database, - schema=source.schema - if source.schema is not None or target_config is None - else target_config.schema, - ) - for source in sources - ) - - -def _loader_to_source_entry( - *, - loader: DiscoveredLoaderFunction, - default_database: str | None, - default_schema: str | None, -) -> SourceEntry: - destination: LoaderDestinationParts = loader_destination_parts( - destination=( - loader.destination if loader.destination is not None else f"__loader__{loader.name}" - ), - default_database=default_database, - default_schema=default_schema, - ) - return SourceEntry( - name=loader.name, - database=destination.database, - schema=destination.schema, - table=destination.table, - loader=loader.name, - write_strategy=loader.write_strategy, - cursor_column=loader.cursor_column, - unique_key=loader.unique_key, - contract=loader.contract, - meta={"sqlbuild_loader_node": True}, - columns=loader.columns, - ) - - -def _parse_load_selector(selector: str) -> tuple[str, bool, bool]: - stripped: str = selector.strip() - include_upstream: bool = stripped.startswith("+") - include_downstream: bool = stripped.endswith("+") - name: str = stripped.strip("+") - return name, include_upstream, include_downstream - - -def _validate_load_selector( - *, - name: str, - managed_sources: dict[str, SourceEntry], - loaders: dict[str, DiscoveredLoaderFunction], -) -> None: - if name in managed_sources or name in loaders: - return - raise CliUserError( - f"sqb load selector '{name}' does not match any managed source or loader", - code="C901", - help="Use exact managed source names or loader function names.", - ) - - -def _upstream_loader_closure( - *, - loader_name: str | None, - upstream_loaders: dict[str, tuple[str, ...]], -) -> set[str]: - if loader_name is None: - return set() - return {loader_name, *transitive_closure(start=loader_name, edges=upstream_loaders)} - - -def _downstream_closure( - *, - loader_name: str | None, - upstream_loaders: dict[str, tuple[str, ...]], - source_by_loader: dict[str, tuple[str, ...]], -) -> tuple[set[str], set[str]]: - if loader_name is None: - return set(), set() - loaders: set[str] = {loader_name} - sources: set[str] = set(source_by_loader.get(loader_name, ())) - downstream_loader: str - deps: tuple[str, ...] - for downstream_loader, deps in upstream_loaders.items(): - if loader_name not in deps: - continue - next_loaders, next_sources = _downstream_closure( - loader_name=downstream_loader, - upstream_loaders=upstream_loaders, - source_by_loader=source_by_loader, - ) - loaders.update(next_loaders) - sources.update(next_sources) - return loaders, sources - - -def _prune_missing_dependencies( - *, - selected_loaders: set[str], - selected_sources: set[str], - managed_sources: dict[str, SourceEntry], - upstream_loaders: dict[str, tuple[str, ...]], - excluded_loaders: set[str], -) -> tuple[set[str], set[str]]: - pruned_loaders: set[str] = set(selected_loaders) - pruned_sources: set[str] = set(selected_sources) - blocked_loaders: set[str] = set(excluded_loaders) - changed: bool = True - while changed: - changed = False - loader_name: str - for loader_name in tuple(pruned_loaders): - if _has_missing_loader_dependency( - loader_name=loader_name, - selected_loaders=pruned_loaders, - upstream_loaders=upstream_loaders, - blocked_loaders=blocked_loaders, - ): - pruned_loaders.remove(loader_name) - blocked_loaders.add(loader_name) - changed = True - source_name: str - for source_name in tuple(pruned_sources): - source_loader: str | None = managed_sources[source_name].loader - if source_loader is not None and source_loader not in pruned_loaders: - pruned_sources.remove(source_name) - changed = True - return pruned_loaders, pruned_sources - - -def _has_missing_loader_dependency( - *, - loader_name: str, - selected_loaders: set[str], - upstream_loaders: dict[str, tuple[str, ...]], - blocked_loaders: set[str], -) -> bool: - dependencies: tuple[str, ...] = upstream_loaders[loader_name] - return any( - dependency not in selected_loaders and dependency in blocked_loaders - for dependency in dependencies - ) - - -def _prune_unneeded_loaders( - *, - selected_loaders: set[str], - selected_sources: set[str], - directly_selected_loaders: set[str], - managed_sources: dict[str, SourceEntry], - upstream_loaders: dict[str, tuple[str, ...]], -) -> set[str]: - needed_loaders: set[str] = set(directly_selected_loaders) - source_name: str - for source_name in selected_sources: - source_loader: str | None = managed_sources[source_name].loader - if source_loader is not None and source_loader in selected_loaders: - needed_loaders.add(source_loader) - - changed: bool = True - while changed: - changed = False - loader_name: str - for loader_name in tuple(needed_loaders): - dependency_name: str - for dependency_name in upstream_loaders.get(loader_name, ()): - if dependency_name in selected_loaders and dependency_name not in needed_loaders: - needed_loaders.add(dependency_name) - changed = True - - return selected_loaders.intersection(needed_loaders) - - -def _topological_loader_order( - *, loader_names: set[str], upstream_loaders: dict[str, tuple[str, ...]] -) -> tuple[str, ...]: - ordered: tuple[str, ...] = () - visited: frozenset[str] = frozenset() - for loader_name in sorted(loader_names): - ordered, visited = _visit_loader( - loader_name=loader_name, - loader_names=loader_names, - upstream_loaders=upstream_loaders, - ordered=ordered, - visited=visited, - ) - return ordered - - -def _visit_loader( - *, - loader_name: str, - loader_names: set[str], - upstream_loaders: dict[str, tuple[str, ...]], - ordered: tuple[str, ...], - visited: frozenset[str], -) -> tuple[tuple[str, ...], frozenset[str]]: - if loader_name in visited: - return ordered, visited - dependency: str - for dependency in upstream_loaders.get(loader_name, ()): - if dependency in loader_names: - ordered, visited = _visit_loader( - loader_name=dependency, - loader_names=loader_names, - upstream_loaders=upstream_loaders, - ordered=ordered, - visited=visited, - ) - return (*ordered, loader_name), visited | {loader_name} diff --git a/src/sqlbuild/cli/commands/_helpers/seed/execution.py b/src/sqlbuild/cli/commands/_helpers/seed/execution.py deleted file mode 100644 index 16b4993020..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/seed/execution.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Seed command execution phases.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from dataclasses import replace -from typing import TextIO - -from sqlbuild._native import ( - ConnectionProgressReporter, - ExecutionEventWriter, - NativeProgressProjector, - current_native_progress_projector, -) -from sqlbuild.cli.commands.models import ( - SeedExecutionPreparation, - SeedInvocation, - SeedRunOutcome, -) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.build.models import SeedExecutionResult -from sqlbuild.executor.build.types import ExecutionStatus -from sqlbuild.executor.pipeline.main.run import run_seed_pipeline -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.coded_error_text import format_coded_error - - -def execute_seed_plan( - *, invocation: SeedInvocation, preparation: SeedExecutionPreparation -) -> SeedRunOutcome: - """Execute compiled seed entries.""" - - start: float = time.monotonic() - event_writer: ExecutionEventWriter = ExecutionEventWriter() - on_complete: Callable[[SeedExecutionResult], None] = _build_on_complete( - stream=invocation.progress_stream, - use_color=invocation.use_color, - seed_order={ - seed_entry.name: index - for index, seed_entry in enumerate( - preparation.pipeline_result.plan_output.seed_entries, start=1 - ) - }, - total_count=len(preparation.pipeline_result.plan_output.seed_entries), - event_writer=event_writer, - plan=preparation.pipeline_result.plan_output, - ) - execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=invocation.adapter_name, - stream=invocation.progress_stream, - blank_line_after_complete=True, - use_color=invocation.use_color, - ) - try: - results: tuple[SeedExecutionResult, ...] = run_seed_pipeline( - plan=preparation.pipeline_result.plan_output, - connection_config=invocation.connection_config, - adapter=invocation.adapter, - max_concurrency=preparation.effective_concurrency, - run_id=preparation.pipeline_result.project.run_id, - query_change_tracking=preparation.pipeline_result.project.settings.query_change_tracking, - on_seed_complete=on_complete, - on_connection_start=execution_connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ) - finally: - event_writer.close() - return SeedRunOutcome(results=results, elapsed=time.monotonic() - start) - - -def _build_on_complete( - *, - stream: TextIO, - use_color: bool, - seed_order: dict[str, int], - total_count: int, - event_writer: ExecutionEventWriter, - plan: PlanOutput, -) -> Callable[[SeedExecutionResult], None]: - projector: NativeProgressProjector | None = current_native_progress_projector() - if projector is not None: - projector.configure_resources(ordinals=seed_order, total=total_count) - - def _on_complete(result: SeedExecutionResult) -> None: - if projector is not None: - canonical_duration_ms: float | None = projector.consume_resource_terminal( - resource_name=result.seed_name, resource_id=f"seed:{result.seed_name}" - ) - if canonical_duration_ms is not None: - result = replace(result, duration_ms=int(canonical_duration_ms)) - status_text: str = "OK" if result.status == ExecutionStatus.SUCCESS else "FAIL" - style: CliStyle = CliStyle(use_color=use_color) - status: str = style.status(status=status_text) - duration: str = "" - if result.duration_ms is not None: - seconds: float = result.duration_ms / 1000.0 - duration = f"{seconds:.2f}s" - ordinal: int = seed_order[result.seed_name] - stream.write( - f" {ordinal}/{total_count} seed {result.seed_name:<30} {status:<6} {duration}\n" - ) - if result.error_message is not None: - stream.write(f" {_format_seed_error(result=result, use_color=use_color)}\n") - stream.flush() - event_writer.write_build_result(result=result, plan=plan, command="seed") - - return _on_complete - - -def _format_seed_error(*, result: SeedExecutionResult, use_color: bool) -> str: - if result.error_message is None: - return "" - if result.error_code is None: - return result.error_message - return format_coded_error( - code=result.error_code, - message=result.error_message, - help=result.error_help, - use_color=use_color, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/seed/invocation.py b/src/sqlbuild/cli/commands/_helpers/seed/invocation.py deleted file mode 100644 index 3fdb7028db..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/seed/invocation.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Seed command invocation resolution phase.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( - resolve_command_invocation, -) -from sqlbuild.cli.commands.models import SeedCommandRequest, SeedInvocation - - -def resolve_seed_invocation(*, request: SeedCommandRequest) -> SeedInvocation: - """Resolve discovery, adapter, connection, and output context for seed.""" - - return resolve_command_invocation(request=request, invocation_type=SeedInvocation) diff --git a/src/sqlbuild/cli/commands/_helpers/seed/outputs.py b/src/sqlbuild/cli/commands/_helpers/seed/outputs.py deleted file mode 100644 index 569630a729..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/seed/outputs.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Seed command output writing and exit-code phases.""" - -from __future__ import annotations - -from sqlbuild._native import ( - format_completion_line, - format_seed_execution_json, - format_summary_footer, - write_execution_header, - write_execution_json_output, -) -from sqlbuild.cli.commands.models import ( - SeedCommandRequest, - SeedExecutionPreparation, - SeedInvocation, - SeedRunOutcome, -) -from sqlbuild.executor.build.models import SeedExecutionResult -from sqlbuild.executor.build.types import ExecutionStatus -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.surface_header import format_surface_header -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.types import CompletionState - - -def write_seed_execution_header( - *, invocation: SeedInvocation, preparation: SeedExecutionPreparation -) -> None: - """Write seed ready, selected seed names, and execution headers.""" - - seed_count: int = len(preparation.pipeline_result.plan_output.seed_entries) - style: CliStyle = CliStyle(use_color=invocation.use_color) - ready_header: str = format_surface_header( - style=style, title="Seed ready", context=f"{seed_count} selected" - ) - invocation.progress_stream.write(f"\n{ready_header}\n\n") - invocation.progress_stream.write(f"{style.section(f'Seeds ({seed_count})')}\n") - seed_index: int - for seed_index, seed_entry in enumerate(preparation.pipeline_result.plan_output.seed_entries): - connector: str = tree_connector(style=style, last=seed_index == seed_count - 1) - invocation.progress_stream.write(f"{connector} {seed_entry.name}\n") - invocation.progress_stream.write("\n") - write_execution_header( - stream=invocation.progress_stream, - command="sqb seed", - target=None, - concurrency=preparation.effective_concurrency, - use_color=invocation.use_color, - ) - - -def write_seed_completion_output( - *, - request: SeedCommandRequest, - invocation: SeedInvocation, - preparation: SeedExecutionPreparation, - outcome: SeedRunOutcome, -) -> None: - """Write seed summary and optional JSON output.""" - - success_count: int = _success_count(outcome.results) - fail_count: int = _fail_count(outcome.results) - counts_summary: str = format_summary_footer( - counts=( - ("PASS", success_count), - ("WARN", 0), - ("FAIL", fail_count), - ("SKIP", 0), - ("TOTAL", len(outcome.results)), - ), - use_color=invocation.use_color, - elapsed=f"{outcome.elapsed:.2f}s", - ) - completion_message: str = format_completion_line( - style=CliStyle(use_color=invocation.use_color), - state=CompletionState.OK if fail_count == 0 else CompletionState.FAIL, - label="Completed successfully" if fail_count == 0 else "Completed with errors", - summary=counts_summary, - ) - invocation.progress_stream.write(f"\n{completion_message}\n") - invocation.progress_stream.flush() - write_execution_json_output( - payload=format_seed_execution_json( - results=outcome.results, - plan=preparation.pipeline_result.plan_output, - ), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def resolve_seed_exit_code(results: tuple[SeedExecutionResult, ...]) -> int: - """Resolve seed exit code from failed results.""" - - return 0 if _fail_count(results) == 0 else 1 - - -def _success_count(results: tuple[SeedExecutionResult, ...]) -> int: - return sum(1 for result in results if result.status == ExecutionStatus.SUCCESS) - - -def _fail_count(results: tuple[SeedExecutionResult, ...]) -> int: - return sum(1 for result in results if result.status == ExecutionStatus.FAILED) diff --git a/src/sqlbuild/cli/commands/_helpers/seed/planning.py b/src/sqlbuild/cli/commands/_helpers/seed/planning.py deleted file mode 100644 index e91cd09bb6..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/seed/planning.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Seed command plan compilation phase.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import ConnectionProgressReporter, build_static_pipeline_result -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.models import ( - SeedCommandRequest, - SeedExecutionPreparation, - SeedInvocation, -) -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, - StaticCommandContext, -) -from sqlbuild.compiler.planner.main.commands.seed import build_seed_command_plan -from sqlbuild.compiler.planner.main.commands.seed_state import read_selected_seed_fingerprints -from sqlbuild.runtime.contracts.main.open_connection import open_connection_with_hooks -from sqlbuild.runtime.contracts.models import ConnectionHooks - - -def prepare_seed_execution( - *, request: SeedCommandRequest, invocation: SeedInvocation -) -> SeedExecutionPreparation: - """Compile seed plan and resolve effective concurrency.""" - - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=invocation.adapter_name, - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - invocation.progress_stream.write("\n") - invocation.progress_stream.flush() - options: CompilePipelineOptions = CompilePipelineOptions( - selected_target=request.selected_target, - select=request.select, - exclude=request.exclude, - connection_config=invocation.connection_config, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ) - context: StaticCommandContext = compile_static_command_context( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - options=options, - ) - selected_seeds: tuple[CompiledSeedView, ...] = tuple( - seed for seed in context.project.seeds if seed.key in context.scope.selected_keys - ) - hooks: ConnectionHooks = ConnectionHooks( - on_connection_start=connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ) - fingerprints: dict[str, Fingerprint] = {} - if selected_seeds: - connection: Any = open_connection_with_hooks( - adapter=invocation.adapter, - connection_config=invocation.connection_config, - hooks=hooks, - ) - try: - fingerprints = read_selected_seed_fingerprints( - adapter=invocation.adapter, - connection=connection, - seeds=selected_seeds, - ) - finally: - invocation.adapter.close(connection) - pipeline_result: CompilePipelineResult = build_static_pipeline_result( - context=context, - plan_output=build_seed_command_plan( - project=context.project, - scope=context.scope, - relations=context.relations, - fingerprints=fingerprints, - ), - ) - effective_concurrency: int = max( - 1, - request.concurrency - if request.concurrency is not None - else pipeline_result.project.settings.concurrency, - ) - return SeedExecutionPreparation( - pipeline_result=pipeline_result, - effective_concurrency=effective_concurrency, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/test/execution.py b/src/sqlbuild/cli/commands/_helpers/test/execution.py deleted file mode 100644 index 8e5d31fd76..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/execution.py +++ /dev/null @@ -1,208 +0,0 @@ -"""Test command execution phases.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import ( - ConnectionProgressReporter, - ExecutionEventWriter, - NativeProgressProjector, - current_native_progress_projector, -) -from sqlbuild.cli.commands._helpers.test.sql_progress import ( - build_test_expectation_rows, - format_parameterized_test_label, - resolve_test_name_width, - test_outcome_status, -) -from sqlbuild.cli.commands.models import ( - TestCommandRequest, - TestExecutionPreparation, - TestInvocation, -) -from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( - NestedCommandProgressCallbacks, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.executor.pipeline.main.run import run_test_pipeline -from sqlbuild.executor.pipeline.models import TestPipelineCallbacks -from sqlbuild.executor.testing.main.resource_id import sql_test_resource_id -from sqlbuild.executor.testing.models import SqlTestExecutionResult -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.surface_header import format_surface_header - - -def prepare_test_execution( - *, - request: TestCommandRequest, - invocation: TestInvocation, - pipeline_result: CompilePipelineResult, -) -> TestExecutionPreparation: - """Prepare nested progress reporting and write the test section header.""" - - test_count: int = len(pipeline_result.plan_output.test_entries) - effective_concurrency: int = resolve_test_concurrency( - request=request, - pipeline_result=pipeline_result, - ) - worker_count: int = min(effective_concurrency, test_count) - model_names: set[str] = set() - for entry in pipeline_result.plan_output.test_entries: - for step in entry.chain: - model_names.add(step.model_name) - model_count: int = len(model_names) - style: CliStyle = CliStyle(use_color=invocation.use_color) - styled_header: str = format_surface_header( - style=style, - title="Test ready", - context=f"{test_count} selected, {model_count} models", - ) - progress: NestedCommandProgressCallbacks = NestedCommandProgressCallbacks( - total=test_count, - label="test", - stream=invocation.progress_stream, - use_color=invocation.use_color, - name_width=resolve_test_name_width(pipeline_result.plan_output.test_entries), - concurrency=worker_count, - ) - projector: NativeProgressProjector | None = current_native_progress_projector() - if projector is not None: - for entry in pipeline_result.plan_output.test_entries: - projector.expect_resource_enrichment(resource_name=entry.name) - invocation.progress_stream.write(f"\n{styled_header}\n\n") - invocation.progress_stream.flush() - execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=invocation.adapter_name, - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - preflight_progress: TransientStatusReporter = TransientStatusReporter( - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - return TestExecutionPreparation( - progress=progress, - execution_connection_progress=execution_connection_progress, - preflight_progress=preflight_progress, - effective_concurrency=effective_concurrency, - worker_count=worker_count, - ) - - -def resolve_test_concurrency( - *, - request: TestCommandRequest, - pipeline_result: CompilePipelineResult, -) -> int: - """Resolve CLI override before the project-wide concurrency setting.""" - - return ( - request.concurrency - if request.concurrency is not None - else pipeline_result.project.settings.concurrency - ) - - -def execute_test_plan( - *, - invocation: TestInvocation, - pipeline_result: CompilePipelineResult, - preparation: TestExecutionPreparation, -) -> tuple[SqlTestExecutionResult, ...]: - """Execute the test plan with nested progress reporting.""" - - event_writer: ExecutionEventWriter = ExecutionEventWriter() - on_complete: Callable[[SqlTestExecutionResult], None] = _build_on_complete( - progress=preparation.progress, - event_writer=event_writer, - pipeline_result=pipeline_result, - ) - try: - return run_test_pipeline( - plan=pipeline_result.plan_output, - connection_config=invocation.connection_config, - adapter=invocation.adapter, - max_concurrency=preparation.effective_concurrency, - callbacks=TestPipelineCallbacks( - on_connection_start=(preparation.execution_connection_progress.on_connection_start), - on_connection_complete=lambda connection_count, elapsed_seconds: ( - preparation.execution_connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - preparation.execution_connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_progress=preparation.preflight_progress.report_preflight_progress, - on_test_start=lambda entry: preparation.progress.on_item_start( - group_name=_test_group_name_from_entry(entry), - item_name=format_parameterized_test_label( - name=entry.name, - source_path=entry.source_path, - parameter_schema=entry.parameter_schema, - parameter_values=entry.parameter_values, - ), - canonical_resource_name=entry.name, - ), - on_test_complete=on_complete, - ), - run_id=pipeline_result.project.run_id, - ) - finally: - event_writer.close() - - -def _build_on_complete( - *, - progress: NestedCommandProgressCallbacks, - event_writer: ExecutionEventWriter, - pipeline_result: CompilePipelineResult, -) -> Callable[[SqlTestExecutionResult], None]: - def _on_complete(result: SqlTestExecutionResult) -> None: - model_name: str = "" - if result.step_results: - model_name = result.step_results[0].model_name - group_name: str = model_name or "(unknown)" - status_text: str = test_outcome_status(outcome=result.outcome) - progress.on_item_complete( - group_name=group_name, - item_name=format_parameterized_test_label( - name=result.test_name, - source_path=result.source_path, - parameter_schema=result.parameter_schema, - parameter_values=result.parameter_values, - ), - status_text=status_text, - child_rows=build_test_expectation_rows(result), - error_code=result.error_code, - error_help=result.error_help, - error_message=result.error_message, - canonical_resource_name=result.test_name, - canonical_resource_id=sql_test_resource_id( - test_name=result.test_name, - source_path=result.source_path, - block_index=result.block_index, - case_name=result.case_name, - ), - ) - event_writer.write_build_result( - result=result, - plan=pipeline_result.plan_output, - command="test", - ) - - return _on_complete - - -def _test_group_name_from_entry(entry: SqlTestPlanEntry) -> str: - for step in entry.chain: - if step.expected_cte_sql is not None: - return step.model_name - if entry.chain: - return entry.chain[0].model_name - return "(unknown)" diff --git a/src/sqlbuild/cli/commands/_helpers/test/inspection.py b/src/sqlbuild/cli/commands/_helpers/test/inspection.py deleted file mode 100644 index e518269e4a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/inspection.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Resolved SQL-test plan inspection output.""" - -from __future__ import annotations - -from collections.abc import Iterable -from typing import TextIO - -from sqlbuild.compiler.planner.models import PlanOutput, PlanWarning, SqlTestPlanEntry -from sqlbuild.compiler.planner.types import WarningSeverity - - -def write_test_plan_inspection(*, stream: TextIO, plan_output: PlanOutput) -> int: - """Write selected test fixture boundaries and real model chains without executing them.""" - - stream.write("\nResolved test plan\n\n") - for entry in plan_output.test_entries: - stream.write(f"{_entry_label(entry)}\n") - _write_names(stream=stream, label="mocked sources", names=entry.mock_source_names) - _write_names(stream=stream, label="mocked refs", names=entry.mock_ref_names) - _write_names(stream=stream, label="mocked seeds", names=entry.mock_seed_names) - _write_names(stream=stream, label="mocked dbt refs", names=entry.mock_dbt_ref_names) - _write_names( - stream=stream, - label="mocked table functions", - names=entry.mock_table_function_names, - ) - _write_names( - stream=stream, - label="real models", - names=tuple(step.model_name for step in entry.chain), - ) - _write_names( - stream=stream, - label="expected models", - names=tuple( - step.model_name for step in entry.chain if step.expected_cte_sql is not None - ), - ) - for mock_name in entry.mock_ref_names: - stream.write(f" boundary: {mock_name} is replaced by __ref__{mock_name}\n") - stream.write("\n") - error_warnings: tuple[PlanWarning, ...] = tuple( - warning for warning in plan_output.warnings if warning.severity == WarningSeverity.ERROR - ) - for warning in plan_output.warnings: - stream.write(f" {warning.severity.value}: {warning.message}\n") - outcome: str = "failed" if error_warnings else "complete" - stream.write( - f"Test plan inspection {outcome}: {len(plan_output.test_entries)} selected, " - f"{len(error_warnings)} errors.\n" - ) - stream.flush() - return 1 if error_warnings else 0 - - -def _write_names(*, stream: TextIO, label: str, names: Iterable[str]) -> None: - values: tuple[str, ...] = tuple(names) - stream.write(f" {label}: {', '.join(values) if values else 'none'}\n") - - -def _entry_label(entry: SqlTestPlanEntry) -> str: - if entry.case_name is None: - return entry.name - return f"{entry.parent_name or entry.name} [{entry.case_name}]" diff --git a/src/sqlbuild/cli/commands/_helpers/test/invocation.py b/src/sqlbuild/cli/commands/_helpers/test/invocation.py deleted file mode 100644 index 562d137d69..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/invocation.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Test command invocation resolution phase.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( - resolve_reported_command_invocation, -) -from sqlbuild.cli.commands.models import TestCommandRequest, TestInvocation - - -def resolve_test_invocation(*, request: TestCommandRequest) -> TestInvocation: - """Resolve test context; ``--inspect`` never connects, so it needs no connection.""" - - return resolve_reported_command_invocation( - request=request, - invocation_type=TestInvocation, - require_connection=not request.inspect, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/test/outputs.py b/src/sqlbuild/cli/commands/_helpers/test/outputs.py deleted file mode 100644 index dca9332ef6..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/outputs.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Test command output writing and exit-code phases.""" - -from __future__ import annotations - -from sqlbuild._native import ( - format_summary_footer, - format_test_execution_json, - write_execution_json_output, -) -from sqlbuild.cli.commands._helpers.test.sql_progress import format_parameterized_test_label -from sqlbuild.cli.commands.models import TestCommandRequest, TestInvocation -from sqlbuild.executor.testing.models import SqlTestExecutionResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.coded_error_text import format_coded_error - -_ERROR_LABEL_WIDTH: int = 10 - - -def write_test_completion_output( - *, - request: TestCommandRequest, - invocation: TestInvocation, - results: tuple[SqlTestExecutionResult, ...], -) -> None: - """Write the test summary footer and optional JSON output.""" - - pass_count: int = sum(1 for r in results if r.outcome == SqlTestOutcome.PASS) - fail_count: int = len(results) - pass_count - summary: str = format_summary_footer( - counts=( - ("PASS", pass_count), - ("FAIL", fail_count), - ("TOTAL", len(results)), - ), - use_color=invocation.use_color, - ) - failure_details: str = _format_test_failure_details( - results=results, - use_color=invocation.use_color, - ) - rendered: str = f"\n{summary}" - if failure_details: - rendered = f"{rendered}\n\n{failure_details}" - invocation.progress_stream.write(f"{rendered}\n") - invocation.progress_stream.flush() - write_execution_json_output( - payload=format_test_execution_json(results=results), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def resolve_test_exit_code(results: tuple[SqlTestExecutionResult, ...]) -> int: - """Resolve the test exit code from failed test outcomes.""" - - pass_count: int = sum(1 for r in results if r.outcome == SqlTestOutcome.PASS) - return 0 if len(results) - pass_count == 0 else 1 - - -def _format_test_failure_details( - *, results: tuple[SqlTestExecutionResult, ...], use_color: bool -) -> str: - failed_results: tuple[SqlTestExecutionResult, ...] = tuple( - result for result in results if result.outcome != SqlTestOutcome.PASS - ) - if not failed_results: - return "" - style: CliStyle = CliStyle(use_color=use_color) - lines: list[str] = [style.error_strong("Failures:"), ""] - result: SqlTestExecutionResult - for result in failed_results: - lines.append( - " " - + format_parameterized_test_label( - name=result.test_name, - source_path=result.source_path, - parameter_schema=result.parameter_schema, - parameter_values=result.parameter_values, - ) - ) - error_message: str = result.error_message or "test failed" - rendered_error: str = error_message - if result.error_code is not None: - rendered_error = format_coded_error( - code=result.error_code, - message=error_message, - help=result.error_help, - use_color=use_color, - include_error_label=False, - ) - label: str = style.error_muted(f"{'error':<{_ERROR_LABEL_WIDTH}}") - error_line: str - for index, error_line in enumerate(rendered_error.splitlines() or [rendered_error]): - display_label: str = label if index == 0 else " " * _ERROR_LABEL_WIDTH - lines.append(f" {display_label}{error_line}") - lines.append("") - return "\n".join(lines).rstrip() diff --git a/src/sqlbuild/cli/commands/_helpers/test/planning.py b/src/sqlbuild/cli/commands/_helpers/test/planning.py deleted file mode 100644 index 7be9b302ba..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/planning.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Test command plan compilation phase.""" - -from __future__ import annotations - -from sqlbuild._native import build_static_pipeline_result -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.models import TestCommandRequest, TestInvocation -from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, - StaticCommandContext, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.commands.sql_test import build_test_command_plan -from sqlbuild.compiler.planner.models import PlanOutput - - -def compile_test_plan( - *, - request: TestCommandRequest, - invocation: TestInvocation, -) -> CompilePipelineResult: - """Compile the test plan for the selected scope.""" - - options: CompilePipelineOptions = CompilePipelineOptions( - selected_target=request.selected_target, - no_sql_validation=request.no_sql_validation, - source_deferral_enabled=False, - select=request.select, - exclude=request.exclude, - accepts_unit_test_selectors=True, - connection_config=invocation.connection_config, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ) - context: StaticCommandContext = compile_static_command_context( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - options=options, - on_progress=invocation.planning_progress.on_progress, - ) - plan_output: PlanOutput = build_test_command_plan( - project=context.project, - adapter=invocation.adapter, - scope=context.scope, - relations=context.relations, - case_name=request.case_name, - ) - if request.case_name is not None: - plan_output = _validate_parameter_case_selection( - plan_output=plan_output, - case_name=request.case_name, - available_cases=plan_output.available_test_case_names, - ) - return build_static_pipeline_result( - context=context, - plan_output=plan_output, - ) - - -def _validate_parameter_case_selection( - *, plan_output: PlanOutput, case_name: str, available_cases: tuple[str, ...] -) -> PlanOutput: - if plan_output.test_entries: - return plan_output - available_text: str = ", ".join(available_cases) or "none" - raise PlannerInputError( - f"SQL test case '{case_name}' did not match the selected tests; available cases: " - f"{available_text}", - code="S008", - ) diff --git a/src/sqlbuild/cli/commands/_helpers/test/sql_progress.py b/src/sqlbuild/cli/commands/_helpers/test/sql_progress.py deleted file mode 100644 index ffce24acfb..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/test/sql_progress.py +++ /dev/null @@ -1,103 +0,0 @@ -"""SQL unit-test progress formatting helpers.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from sqlbuild._native import ( - format_expectation_detail, - format_expectation_name, - resolve_name_column_width, -) -from sqlbuild.cli.progress.constants import NESTED_CHILD_NAME_INDENT -from sqlbuild.cli.progress.models import NestedProgressChildRow -from sqlbuild.compiler.discovery.models import SqlTestParameterDeclaration -from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind - - -def resolve_test_name_width(test_entries: tuple[SqlTestPlanEntry, ...]) -> int: - """Resolve one shared width for test rows and nested expectation rows.""" - - names: list[str] = [] - child_names: list[str] = [] - entry: SqlTestPlanEntry - for entry in test_entries: - names.append( - format_parameterized_test_label( - name=entry.name, - source_path=entry.source_path, - parameter_schema=entry.parameter_schema, - parameter_values=entry.parameter_values, - ) - ) - child_names.extend( - f"expected {step.model_name}" for step in entry.chain if step.expected_cte_sql - ) - child_names.extend(f"assertion {assertion.name}" for assertion in entry.assertions) - child_name_length: int = max((len(name) for name in child_names), default=0) - return max( - resolve_name_column_width(names=names, min_width=50), - child_name_length + NESTED_CHILD_NAME_INDENT, - ) - - -def build_test_expectation_rows( - result: SqlTestExecutionResult, -) -> tuple[NestedProgressChildRow, ...]: - """Build nested expectation rows for a completed SQL unit test.""" - - rows: list[NestedProgressChildRow] = [] - step_result: StepResult - for step_result in result.step_results: - expectation_name: str = format_expectation_name(step_result.model_name) - status_text: str = test_outcome_status(outcome=step_result.outcome) - rows.append( - NestedProgressChildRow( - label="expect", - name=expectation_name, - status_text=status_text, - detail=format_expectation_detail(step_result), - ) - ) - return tuple(rows) - - -def format_parameterized_test_label( - *, - name: str, - source_path: Path | None, - parameter_schema: tuple[SqlTestParameterDeclaration, ...], - parameter_values: tuple[tuple[str, SqlValue], ...], -) -> str: - """Add source and safe typed values only for parameterized test rows.""" - - if source_path is None or not parameter_values: - return name - schema_by_name: dict[str, SqlTestParameterDeclaration] = { - parameter.name: parameter for parameter in parameter_schema - } - rendered_values: list[str] = [] - value_name: str - value: SqlValue - for value_name, value in parameter_values: - declaration: SqlTestParameterDeclaration = schema_by_name[value_name] - payload: object = str(value.value) if value.kind == SqlValueKind.DECIMAL else value.value - rendered: str = json.dumps(payload, ensure_ascii=True, separators=(",", ":")) - nullable: str = "?" if declaration.nullable else "" - rendered_values.append(f"{value_name}:{declaration.value_type.value}{nullable}={rendered}") - return f"{name} ({source_path.as_posix()}; {', '.join(rendered_values)})" - - -def test_outcome_status(*, outcome: SqlTestOutcome) -> str: - """Preserve the semantic distinction between assertion failures and execution errors.""" - - if outcome == SqlTestOutcome.PASS: - return "PASS" - if outcome == SqlTestOutcome.ERROR: - return "ERROR" - return "FAIL" diff --git a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py index eb5a92b134..8beb7512a2 100644 --- a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py +++ b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py @@ -16,12 +16,12 @@ format_completion_line, format_expectation_detail, format_expectation_name, + format_parameterized_test_label, format_status_cell, format_summary_footer, ) from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.commands._helpers.test.sql_progress import format_parameterized_test_label from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.types import CompiledResourceType diff --git a/src/sqlbuild/cli/commands/classes/load_progress_reporter.py b/src/sqlbuild/cli/commands/classes/load_progress_reporter.py deleted file mode 100644 index b3ed384d04..0000000000 --- a/src/sqlbuild/cli/commands/classes/load_progress_reporter.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Progress callbacks for sqb load execution.""" - -from __future__ import annotations - -import threading -from typing import TextIO - -from sqlbuild._native import ( - NativeProgressProjector, - current_native_progress_projector, - format_completion_line, - format_summary_footer, -) -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator -from sqlbuild.presentation.types import CompletionState -from sqlbuild.spec.contracts.models import SourceEntry - -_SPINNER_TICK_SECONDS: float = 0.1 -_ACTIVE_SPINNER_FRAMES: tuple[str, ...] = ( - "⠋", - "⠙", - "⠹", - "⠸", - "⠼", - "⠴", - "⠦", - "⠧", - "⠇", - "⠏", -) - - -class LoadProgressReporter: - """TTY-aware renderer for sqb load execution progress.""" - - def __init__( - self, - *, - stream: TextIO, - use_color: bool, - source_order: dict[str, int], - total_count: int, - ) -> None: - self._stream: TextIO = stream - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._style: CliStyle = CliStyle(use_color=use_color) - self._source_order: dict[str, int] = source_order - self._total_count: int = total_count - self._is_tty: bool = stream.isatty() - self._current_source: SourceEntry | None = None - self._current_sub_message: str = "" - self._spinner_frame_index: int = 0 - self._spinner_stop_event: threading.Event | None = None - self._spinner_thread: threading.Thread | None = None - self._write_lock: threading.RLock = self._lines.lock - self._spinner_line_active: bool = False - self._cursor_hidden: bool = False - self._projector: NativeProgressProjector | None = current_native_progress_projector() - if self._projector is not None: - self._projector.configure_resources(ordinals=source_order, total=total_count) - - def on_start(self, source: SourceEntry) -> None: - if self._projector is not None: - return - if not self._is_tty: - return - self._current_source = source - self._current_sub_message = "" - self._hide_cursor() - with self._write_lock: - self._spinner_line_active = True - self._lines.claim(stream=self._stream, owner=self) - self._write_spinner_line() - self._start_spinner_loop() - - def on_progress(self, *, source: SourceEntry, message: str) -> None: - if not self._is_tty or self._current_source is None: - return - if source.name != self._current_source.name: - return - self._current_sub_message = message - self._write_spinner_line() - - def on_complete(self, result: LoadExecutionResult) -> None: - canonical_duration_ms: float | None = None - if self._projector is not None: - canonical_duration_ms = self._projector.consume_resource_terminal( - resource_name=result.source_name, - resource_id=f"source:{result.source_name}", - ) - if canonical_duration_ms is None: - return - self._stop_spinner_loop() - if self._is_tty: - with self._write_lock: - self._spinner_line_active = False - self._lines.release(owner=self) - self._stream.write("\r\033[K") - self._stream.flush() - self._show_cursor() - status_text: str = ( - "OK" - if result.status == ExecutionStatus.SUCCESS - else "SKIP" - if result.status == ExecutionStatus.SKIPPED - else "FAIL" - ) - status: str = self._style.status(status=status_text) - duration: str = "" - duration_ms: float | None = ( - result.duration_ms if canonical_duration_ms is None else canonical_duration_ms - ) - if duration_ms is not None: - duration = f"{duration_ms / 1000.0:.2f}s" - rows_loaded: str = f"rows={result.rows_loaded:,}" - ordinal: int = self._source_order[result.source_name] - self._stream.write( - f" {ordinal}/{self._total_count} {result.resource_kind.value:<10}" - f"{result.source_name:<30} {status:<6} {duration} {rows_loaded}\n" - ) - event: LifeCycleEvent - for event in result.lifecycle_events: - if event.kind == LifeCycleEventKind.LOG: - self._write_log_block(event.content) - if result.error_message is not None: - self._stream.write(self._style.error(f" {result.error_message}\n")) - self._stream.flush() - - def clear_transient_line(self) -> None: - """Erase the live spinner row so a persistent line can take its place.""" - - with self._write_lock: - self._stream.write("\r\033[K") - self._stream.flush() - - def redraw_transient_line(self) -> None: - """Draw the live spinner row again below persistent output.""" - - self._write_spinner_line() - - def _write_spinner_line(self) -> None: - with self._write_lock: - if self._spinner_line_active: - self._draw_spinner_line() - - def _draw_spinner_line(self) -> None: - source: SourceEntry | None = self._current_source - if source is None: - return - ordinal: int = self._source_order[source.name] - status: str = self._style.status(status=_ACTIVE_SPINNER_FRAMES[self._spinner_frame_index]) - self._spinner_frame_index = (self._spinner_frame_index + 1) % len(_ACTIVE_SPINNER_FRAMES) - resource_kind: str = ( - "loader" if source.meta.get("sqlbuild_loader_node") is True else "source" - ) - name_display: str = source.name - if self._current_sub_message: - name_display = f"{source.name} {self._current_sub_message}" - line: str = ( - f" {ordinal}/{self._total_count} {resource_kind:<10}{name_display:<48} {status}" - ) - self._stream.write(f"\r\033[K{line}") - self._stream.flush() - - def _start_spinner_loop(self) -> None: - self._stop_spinner_loop() - stop_event: threading.Event = threading.Event() - self._spinner_stop_event = stop_event - spinner_thread: threading.Thread = threading.Thread( - target=self._spin_until_stopped, - args=(stop_event,), - daemon=True, - ) - self._spinner_thread = spinner_thread - spinner_thread.start() - - def _stop_spinner_loop(self) -> None: - if self._spinner_stop_event is not None: - self._spinner_stop_event.set() - if self._spinner_thread is not None and self._spinner_thread.is_alive(): - self._spinner_thread.join(timeout=0.2) - self._spinner_stop_event = None - self._spinner_thread = None - - def _spin_until_stopped(self, stop_event: threading.Event) -> None: - while not stop_event.wait(_SPINNER_TICK_SECONDS): - self._write_spinner_line() - - def _hide_cursor(self) -> None: - if self._cursor_hidden: - return - with self._write_lock: - self._stream.write("\033[?25l") - self._stream.flush() - self._cursor_hidden = True - - def _show_cursor(self) -> None: - if not self._cursor_hidden: - return - with self._write_lock: - self._stream.write("\033[?25h") - self._stream.flush() - self._cursor_hidden = False - - def _write_log_block(self, message: str) -> None: - lines: list[str] = message.splitlines() or [""] - prefix: str = self._style.log_label(" log ") - first_content: str = self._style.muted(lines[0]) - self._stream.write(f"\n{prefix}{first_content}\n") - line: str - for line in lines[1:]: - self._stream.write(f"{self._style.muted(f' {line}')}\n") - - -def format_load_footer( - *, - results: tuple[LoadExecutionResult, ...], - success_count: int, - warn_count: int, - fail_count: int, - skip_count: int, - elapsed: float, - use_color: bool, -) -> str: - style: CliStyle = CliStyle(use_color=use_color) - counts_summary: str = format_summary_footer( - counts=( - ("PASS", success_count), - ("WARN", warn_count), - ("FAIL", fail_count), - ("SKIP", skip_count), - ("TOTAL", len(results)), - ), - use_color=use_color, - elapsed=f"{elapsed:.2f}s", - ) - state: CompletionState = CompletionState.OK - label: str = "Completed successfully" - if fail_count: - state, label = CompletionState.FAIL, "Completed with errors" - elif warn_count: - state, label = CompletionState.WARN, "Completed with warnings" - completion_message: str = format_completion_line( - style=style, state=state, label=label, summary=counts_summary - ) - return "".join(_load_warning_lines(results=results, style=style)) + f"\n{completion_message}\n" - - -def _load_warning_lines(*, results: tuple[LoadExecutionResult, ...], style: CliStyle) -> list[str]: - lines: list[str] = [] - result: LoadExecutionResult - for result in results: - if not result.warning_messages: - continue - if not lines: - lines.append(f"\n{style.warning_strong('Warnings:')}\n\n") - lines.append(f" {result.source_name} (loader {result.loader_name})\n") - lines.extend(f" {message}\n" for message in result.warning_messages) - return lines diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index 22b5406d2c..03e718d228 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -65,13 +65,6 @@ SCOPE_GLOBAL_SUMMARY: str = "summary" SCOPE_GLOBAL_ALL: str = "all" SCOPE_DEFAULT_PAGE_SIZE: int = 100 -READ_ONLY_QUERY_ROOT_KEYS: frozenset[str] = frozenset( - {"except", "intersect", "select", "union", "values"} -) -DEFAULT_DIFF_MAX_VALUE_LENGTH: int = 160 -QUERY_DIFF_INCOMPLETE_PREPARATION_CODES: frozenset[str] = frozenset( - {"C229", "C230", "C231", "C232", "C233", "C234", "C235", "C240", "C244"} -) MISSING_ORIGIN_BUILD_HELP: str = ( "Pass --allow-missing-migration-origin to confirm in non-interactive runs." ) @@ -106,7 +99,6 @@ ) EXECUTION_LIMIT_BUILD_NOTE: str = "No warehouse changes were made." EXECUTION_LIMIT_DBT_NOTE: str = "dbt work has already run; no SQLBuild models were built." -QUERY_DIFF_INCOMPLETE_EXECUTION_CODES: frozenset[str] = frozenset({"C237", "C238", "C239"}) CLI_WAREHOUSE_SOURCE: str = "--warehouse" CONNECTION_WAREHOUSE_SOURCE: str = "connection warehouse" CONNECTION_DEFAULT_WAREHOUSE_SOURCE: str = "connection default" diff --git a/src/sqlbuild/cli/commands/main/dbt/_dbt.py b/src/sqlbuild/cli/commands/main/dbt/_dbt.py index 6e8e39dd0c..705c0e75a6 100644 --- a/src/sqlbuild/cli/commands/main/dbt/_dbt.py +++ b/src/sqlbuild/cli/commands/main/dbt/_dbt.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import PlanningProgressReporter +from sqlbuild._native import PlanningProgressReporter, render_dbt_interop_plan from sqlbuild.cli.commands._helpers.dbt.auto_init import ( ensure_sqlbuild_project_for_dbt_command, ) @@ -19,9 +19,6 @@ from sqlbuild.integrations.dbt.main.cli.validate_execution_args import validate_dbt_execution_args from sqlbuild.integrations.dbt.main.pipeline.execute import execute_dbt_interop_from_project from sqlbuild.integrations.dbt.main.pipeline.plan import plan_dbt_interop_from_project -from sqlbuild.integrations.dbt.main.pipeline.render_plan import ( - render_dbt_interop_plan, -) from sqlbuild.integrations.dbt.models import DbtInteropExecutionRequest, DbtInteropPlan from sqlbuild.integrations.dbt.types import DbtInteropCommand from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/main/execution/_check.py b/src/sqlbuild/cli/commands/main/execution/_check.py deleted file mode 100644 index df13f84336..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_check.py +++ /dev/null @@ -1,66 +0,0 @@ -"""CLI check command entry point for Python checks.""" - -from __future__ import annotations - -from sqlbuild._native import write_execution_header -from sqlbuild.cli.commands._helpers.check.execution import ( - execute_check_plan, - prepare_check_execution, -) -from sqlbuild.cli.commands._helpers.check.invocation import resolve_check_invocation -from sqlbuild.cli.commands._helpers.check.outputs import ( - resolve_check_exit_code, - write_check_completion_output, -) -from sqlbuild.cli.commands._helpers.check.planning import compile_check_plan -from sqlbuild.cli.commands.models import ( - CheckCommandRequest, - CheckExecutionPreparation, - CheckInvocation, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult -from sqlbuild.provider.classes.session import ProviderSession -from sqlbuild.provider.main.session import build_provider_session - - -def run_check(request: CheckCommandRequest) -> int: - """Execute the check command.""" - - invocation: CheckInvocation = resolve_check_invocation(request=request) - invocation.progress_stream.write("\n") - write_execution_header( - stream=invocation.progress_stream, - command="sqb check", - target=None, - concurrency=1, - use_color=invocation.use_color, - ) - pipeline_result: CompilePipelineResult = compile_check_plan( - request=request, - invocation=invocation, - ) - preparation: CheckExecutionPreparation = prepare_check_execution( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - ) - provider_session: ProviderSession = build_provider_session( - discovered_providers=invocation.discovered_inputs.providers - ) - try: - results: tuple[PythonCheckExecutionResult, ...] = execute_check_plan( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - providers=provider_session.providers, - ) - write_check_completion_output( - request=request, - invocation=invocation, - preparation=preparation, - results=results, - ) - return resolve_check_exit_code(results) - finally: - provider_session.close() diff --git a/src/sqlbuild/cli/commands/main/execution/_load.py b/src/sqlbuild/cli/commands/main/execution/_load.py deleted file mode 100644 index a561c754f7..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_load.py +++ /dev/null @@ -1,47 +0,0 @@ -"""CLI load command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.load.execution import execute_load_plan -from sqlbuild.cli.commands._helpers.load.invocation import resolve_load_invocation -from sqlbuild.cli.commands._helpers.load.outputs import ( - resolve_load_exit_code, - write_empty_load_output, - write_load_completion_output, - write_load_execution_header, - write_load_plan_output, - write_load_ready_output, -) -from sqlbuild.cli.commands._helpers.load.planning import prepare_load_execution -from sqlbuild.cli.commands.models import ( - LoadCommandRequest, - LoadExecutionPreparation, - LoadInvocation, - LoadRunOutcome, -) - - -def run_load(request: LoadCommandRequest) -> int: - """Execute the load command.""" - - invocation: LoadInvocation = resolve_load_invocation(request=request) - write_load_ready_output(invocation=invocation) - if not invocation.selected_sources: - write_empty_load_output(request=request, invocation=invocation) - return 0 - preparation: LoadExecutionPreparation = prepare_load_execution( - request=request, - invocation=invocation, - ) - try: - write_load_plan_output(invocation=invocation) - write_load_execution_header(invocation=invocation, preparation=preparation) - outcome: LoadRunOutcome = execute_load_plan( - request=request, - invocation=invocation, - preparation=preparation, - ) - finally: - preparation.provider_session.close() - write_load_completion_output(request=request, invocation=invocation, outcome=outcome) - return resolve_load_exit_code(outcome) diff --git a/src/sqlbuild/cli/commands/main/execution/_seed.py b/src/sqlbuild/cli/commands/main/execution/_seed.py deleted file mode 100644 index c7261e2a56..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_seed.py +++ /dev/null @@ -1,40 +0,0 @@ -"""CLI seed command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.seed.execution import execute_seed_plan -from sqlbuild.cli.commands._helpers.seed.invocation import resolve_seed_invocation -from sqlbuild.cli.commands._helpers.seed.outputs import ( - resolve_seed_exit_code, - write_seed_completion_output, - write_seed_execution_header, -) -from sqlbuild.cli.commands._helpers.seed.planning import prepare_seed_execution -from sqlbuild.cli.commands.models import ( - SeedCommandRequest, - SeedExecutionPreparation, - SeedInvocation, - SeedRunOutcome, -) - - -def run_seed(request: SeedCommandRequest) -> int: - """Execute the seed command.""" - - invocation: SeedInvocation = resolve_seed_invocation(request=request) - preparation: SeedExecutionPreparation = prepare_seed_execution( - request=request, - invocation=invocation, - ) - write_seed_execution_header(invocation=invocation, preparation=preparation) - outcome: SeedRunOutcome = execute_seed_plan( - invocation=invocation, - preparation=preparation, - ) - write_seed_completion_output( - request=request, - invocation=invocation, - preparation=preparation, - outcome=outcome, - ) - return resolve_seed_exit_code(outcome.results) diff --git a/src/sqlbuild/cli/commands/main/execution/_test.py b/src/sqlbuild/cli/commands/main/execution/_test.py deleted file mode 100644 index 52114df624..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_test.py +++ /dev/null @@ -1,70 +0,0 @@ -"""CLI test command entry point.""" - -from __future__ import annotations - -from sqlbuild._native import write_execution_header -from sqlbuild.cli.commands._helpers.test.execution import ( - execute_test_plan, - prepare_test_execution, - resolve_test_concurrency, -) -from sqlbuild.cli.commands._helpers.test.inspection import write_test_plan_inspection -from sqlbuild.cli.commands._helpers.test.invocation import resolve_test_invocation -from sqlbuild.cli.commands._helpers.test.outputs import ( - resolve_test_exit_code, - write_test_completion_output, -) -from sqlbuild.cli.commands._helpers.test.planning import compile_test_plan -from sqlbuild.cli.commands.models import ( - TestCommandRequest, - TestExecutionPreparation, - TestInvocation, -) -from sqlbuild.cli.target_artifacts.main._write_test_runtime_target import write_test_runtime_target -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.executor.testing.models import SqlTestExecutionResult - - -def run_test(request: TestCommandRequest) -> int: - """Execute the test command.""" - - invocation: TestInvocation = resolve_test_invocation(request=request) - pipeline_result: CompilePipelineResult = compile_test_plan( - request=request, - invocation=invocation, - ) - if request.inspect: - return write_test_plan_inspection( - stream=invocation.progress_stream, - plan_output=pipeline_result.plan_output, - ) - invocation.progress_stream.write("\n") - write_execution_header( - stream=invocation.progress_stream, - command="sqb test", - target=None, - concurrency=resolve_test_concurrency(request=request, pipeline_result=pipeline_result), - use_color=invocation.use_color, - ) - preparation: TestExecutionPreparation = prepare_test_execution( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - ) - results: tuple[SqlTestExecutionResult, ...] = execute_test_plan( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - ) - write_test_runtime_target( - target_dir=invocation.effective_project_dir / "target", - adapter=invocation.adapter, - plan_output=pipeline_result.plan_output, - results=results, - ) - write_test_completion_output( - request=request, - invocation=invocation, - results=results, - ) - return resolve_test_exit_code(results) diff --git a/src/sqlbuild/cli/commands/main/inspection/_clone.py b/src/sqlbuild/cli/commands/main/inspection/_clone.py deleted file mode 100644 index 996a76626b..0000000000 --- a/src/sqlbuild/cli/commands/main/inspection/_clone.py +++ /dev/null @@ -1,60 +0,0 @@ -"""CLI clone command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.clone.connections import ( - close_clone_targets, - connect_clone_targets, -) -from sqlbuild.cli.commands._helpers.clone.execution import execute_clone_plan -from sqlbuild.cli.commands._helpers.clone.invocation import resolve_clone_invocation -from sqlbuild.cli.commands._helpers.clone.outputs import ( - resolve_clone_exit_code, - write_clone_completion_output, - write_clone_execution_header, - write_clone_execution_json_output, -) -from sqlbuild.cli.commands._helpers.clone.planning import prepare_clone_execution -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneConnectionContext, - CloneExecutionPreparation, - CloneInvocation, - CloneRunOutcome, -) - - -def run_clone(request: CloneCommandRequest) -> int: - """Execute the clone command.""" - - invocation: CloneInvocation = resolve_clone_invocation(request=request) - connection_context: CloneConnectionContext = connect_clone_targets( - request=request, - invocation=invocation, - ) - try: - preparation: CloneExecutionPreparation = prepare_clone_execution( - request=request, - invocation=invocation, - connection_context=connection_context, - ) - write_clone_execution_header( - request=request, - invocation=invocation, - preparation=preparation, - ) - outcome: CloneRunOutcome = execute_clone_plan( - request=request, - invocation=invocation, - connection_context=connection_context, - preparation=preparation, - ) - finally: - _ = close_clone_targets(invocation=invocation, connection_context=connection_context) - write_clone_completion_output(invocation=invocation, outcome=outcome) - write_clone_execution_json_output( - request=request, - preparation=preparation, - outcome=outcome, - ) - return resolve_clone_exit_code(outcome) diff --git a/src/sqlbuild/cli/commands/main/inspection/_diff.py b/src/sqlbuild/cli/commands/main/inspection/_diff.py deleted file mode 100644 index e48ffabe89..0000000000 --- a/src/sqlbuild/cli/commands/main/inspection/_diff.py +++ /dev/null @@ -1,148 +0,0 @@ -"""CLI diff command entry point.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.cli.commands._helpers.diff.execution import ( - execute_direct_diff, - execute_query_diff, - prepare_direct_diff, - prepare_query_diff, -) -from sqlbuild.cli.commands._helpers.diff.invocation import ( - is_query_diff_request, - resolve_diff_invocation, -) -from sqlbuild.cli.commands._helpers.diff.outputs import ( - resolve_diff_exit_code, - write_direct_diff_output, -) -from sqlbuild.cli.commands.constants import ( - QUERY_DIFF_INCOMPLETE_EXECUTION_CODES, - QUERY_DIFF_INCOMPLETE_PREPARATION_CODES, -) -from sqlbuild.cli.commands.exceptions import ( - CliUserError, - QueryDiffExecutionError, - QueryDiffIncompleteError, - QueryDiffOutcomeError, -) -from sqlbuild.cli.commands.models import ( - DiffCommandRequest, - DiffInvocation, - DirectDiffPreparation, - QueryDiffPreparation, - QueryDiffRunOutcome, -) -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.diff.constants import ( - ROW_DIFF_KEY_ERROR_FRAGMENT, - ROW_DIFF_KEY_ERROR_PREFIX, -) -from sqlbuild.executor.diff.models import DiffExecutionResult - - -def run_diff(request: DiffCommandRequest) -> int: - """Execute the diff command.""" - - query_mode: bool = is_query_diff_request(request=request) - try: - invocation: DiffInvocation = resolve_diff_invocation(request=request) - except QueryDiffOutcomeError: - raise - except Exception as error: - if query_mode: - raise QueryDiffExecutionError(str(error), code="C252") from error - raise - if query_mode: - try: - query_preparation: QueryDiffPreparation = prepare_query_diff( - request=request, - invocation=invocation, - ) - except CliUserError as error: - error_type: type[QueryDiffOutcomeError] = ( - QueryDiffIncompleteError - if error.code in QUERY_DIFF_INCOMPLETE_PREPARATION_CODES - else QueryDiffExecutionError - ) - raise error_type(error.message, code=error.code, help=error.help) from error - except Exception as error: - raise QueryDiffExecutionError(str(error), code="C253") from error - try: - query_outcome: QueryDiffRunOutcome = execute_query_diff( - request=request, - preparation=query_preparation, - ) - except QueryDiffOutcomeError: - raise - except ExecutorInputError as error: - raise QueryDiffIncompleteError( - str(error), - code=str(getattr(error, "code", "C254")), - help=getattr(error, "help", None), - ) from error - except CliUserError as error: - error_type = ( - QueryDiffIncompleteError - if error.code in QUERY_DIFF_INCOMPLETE_EXECUTION_CODES - else QueryDiffExecutionError - ) - raise error_type(error.message, code=error.code, help=error.help) from error - except AdapterUserError as error: - error_type = ( - QueryDiffIncompleteError - if _is_query_key_preflight_error(error=error) - else QueryDiffExecutionError - ) - raise error_type(error.message, code=error.code, help=error.help) from error - except Exception as error: - raise QueryDiffExecutionError(str(error), code="C255") from error - try: - write_direct_diff_output( - request=request, - preparation=DirectDiffPreparation( - from_target=query_preparation.left_label, - to_target=query_preparation.right_label, - adapter=query_preparation.adapter, - left_project=None, - right_project=None, - selected_names=( - f"{query_preparation.left_label} vs {query_preparation.right_label}", - ), - connection_config=query_preparation.connection_config, - effective_max_column_examples=query_preparation.effective_max_column_examples, - effective_max_row_only_examples=( - query_preparation.effective_max_row_only_examples - ), - ), - result=query_outcome.result, - phase_seconds=query_outcome.phase_seconds, - outcome=query_outcome.outcome, - ) - except Exception as error: - raise QueryDiffExecutionError( - f"failed to publish query diff output: {error}", code="C257" - ) from error - return query_outcome.exit_code - - direct_preparation: DirectDiffPreparation = prepare_direct_diff( - request=request, - invocation=invocation, - ) - result: DiffExecutionResult = execute_direct_diff( - request=request, - preparation=direct_preparation, - ) - write_direct_diff_output( - request=request, - preparation=direct_preparation, - result=result, - ) - return resolve_diff_exit_code(result) - - -def _is_query_key_preflight_error(*, error: AdapterUserError) -> bool: - return error.message.startswith(ROW_DIFF_KEY_ERROR_PREFIX) and ( - ROW_DIFF_KEY_ERROR_FRAGMENT in error.message - ) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 654d4ccd72..3429c98306 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Any, TextIO +from typing import TYPE_CHECKING, TextIO from sqlbuild.cli.commands.types import ( PlaygroundTemplate, @@ -36,7 +36,6 @@ CostConfig, ExecutionLimitsConfig, SnapshotsConfig, - SourceEntry, ) if TYPE_CHECKING: @@ -50,32 +49,23 @@ DirectPythonLifecycleState, ) from sqlbuild.cli.progress.classes.audit_progress_reporter import AuditProgressReporter - from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( - NestedCommandProgressCallbacks, - ) from sqlbuild.compiler.compile.models import ( CompiledSqlScenario, CompilerDiagnostic, ) from sqlbuild.compiler.compile.types import CompiledProjectView - from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs + from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ( - ClonePipelineResult, CompilePipelineResult, PythonPlanEntry, ) from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput - from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan from sqlbuild.compiler.refactoring.models import RefactorProject - from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult - from sqlbuild.executor.clone.models import CloneExecutionResult - from sqlbuild.executor.diff.models import DiffExecutionResult, FullDiffSizeLimits + from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.janitor.models import JanitorPlan - from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.integrations.dbt.models import DbtInitRequest from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter - from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.runtime.contracts.types import NodeStartCallback @@ -305,20 +295,6 @@ class CheckInvocation: planning_progress: PlanningProgressReporter -@dataclass(frozen=True) -class CheckExecutionPreparation: - """Prepared Python graph, check selection, lifecycle, and relation defaults.""" - - python_graph: PythonNodeGraph - check_functions: tuple[DiscoveredCheckFunction, ...] - lifecycle_plan: PythonSqlRunLifecyclePlan - relation_targets: dict[SqlResourceRef, str] - relation_refs: frozenset[SqlResourceRef] - default_database: str | None - default_schema: str | None - project_relations: dict[SqlResourceRef, str] | None = None - - @dataclass(frozen=True) class CloneCommandRequest: """CLI inputs for one clone command invocation.""" @@ -337,42 +313,6 @@ class CloneCommandRequest: event_output_path: Path | None = None -@dataclass(frozen=True) -class CloneInvocation: - """Resolved project, adapter, and output context for clone.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter_name: str - adapter: BaseAdapter - destination_target_name: str - use_color: bool - progress_stream: TextIO - - -@dataclass(frozen=True) -class CloneConnectionContext: - """Destination connection configuration and handle for a clone.""" - - destination_connection_config: dict[str, object] - destination_connection: Any - - -@dataclass(frozen=True) -class CloneExecutionPreparation: - """Prepared direct clone pipeline and selected destination entries.""" - - pipeline_result: ClonePipelineResult - - -@dataclass(frozen=True) -class CloneRunOutcome: - """Direct clone execution result and elapsed time.""" - - result: CloneExecutionResult - elapsed: float - - @dataclass(frozen=True) class DbtSqlbuildWorkContext: """Shared execution context for one dbt interop SQLBuild work run.""" @@ -466,30 +406,6 @@ class DiffCommandRequest: full_example_values: bool = False -@dataclass(frozen=True) -class DiffInvocation: - """Resolved project discovery and mode for diff.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - - -@dataclass(frozen=True) -class DirectDiffPreparation: - """Resolved direct diff adapter and compiled target projects.""" - - from_target: str - to_target: str - adapter: BaseAdapter - left_project: Any - right_project: Any - selected_names: tuple[str, ...] - connection_config: dict[str, object] - effective_max_column_examples: int - effective_max_row_only_examples: int - full_size_limits: FullDiffSizeLimits | None = None - - @dataclass(frozen=True) class QueryDiffPreparation: """Resolved raw-query diff adapter, connection, SQL, and limits.""" @@ -509,35 +425,6 @@ class QueryDiffPreparation: effective_max_row_only_examples: int -@dataclass(frozen=True) -class DiffExampleRenderOptions: - """Bounded example rendering controls independent of comparison execution.""" - - max_value_length: int | None - suppress_values: bool = False - - -@dataclass(frozen=True) -class RenderedDiffExampleValue: - """One safely bounded value plus explicit information-loss metadata.""" - - text: str | None - original_length: int - truncated: bool - suppressed: bool - first_difference: int | None - - -@dataclass(frozen=True) -class QueryDiffRunOutcome: - """Raw-query result plus measured lifecycle phase durations.""" - - result: DiffExecutionResult - phase_seconds: dict[str, float] - outcome: str - exit_code: int - - @dataclass(frozen=True) class FreshnessCommandRequest: """CLI inputs for one `sqb freshness` invocation.""" @@ -633,64 +520,6 @@ class LoadCommandRequest: json_output_path: Path | None = None -@dataclass(frozen=True) -class LoadInvocation: - """Resolved project, selected sources, and output context for load.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - selected_sources: tuple[SourceEntry, ...] - reference_sources: tuple[SourceEntry, ...] - use_color: bool - progress_stream: TextIO - relation_sources: tuple[SourceEntry, ...] - - -@dataclass(frozen=True) -class LoadExecutionPreparation: - """Prepared adapter, connection, runtime, and execution settings.""" - - adapter_name: str - adapter: BaseAdapter - connection_config: dict[str, object] - target_name: str | None - effective_vars: dict[str, object] - run_id: str - effective_cursor_overrides: CursorOverrides - effective_concurrency: int - provider_session: Any - project_relations: dict[SqlResourceRef, str] | None - - -@dataclass(frozen=True) -class LoadRunOutcome: - """Load execution results, elapsed time, and summary counts.""" - - results: tuple[LoadExecutionResult, ...] - elapsed: float - success_count: int - fail_count: int - skip_count: int - warn_count: int - - -@dataclass(frozen=True) -class LoadSelectionSets: - """Mutable selection sets returned explicitly by selection phases.""" - - selected_sources: set[str] - selected_loaders: set[str] - - -@dataclass(frozen=True) -class LoadSelectorSets: - """Selection sets updated while applying include selectors.""" - - selected_sources: set[str] - selected_loaders: set[str] - directly_selected_loaders: set[str] - - @dataclass(frozen=True) class PlanCommandRequest: """CLI inputs for one plan command invocation.""" @@ -870,22 +699,6 @@ class SeedInvocation: progress_stream: TextIO -@dataclass(frozen=True) -class SeedExecutionPreparation: - """Prepared seed execution settings and compiled pipeline.""" - - pipeline_result: CompilePipelineResult - effective_concurrency: int - - -@dataclass(frozen=True) -class SeedRunOutcome: - """Seed execution results and elapsed time.""" - - results: tuple[SeedExecutionResult, ...] - elapsed: float - - @dataclass(frozen=True) class TestCommandRequest: """CLI inputs for one test command invocation.""" @@ -970,17 +783,6 @@ class TestInvocation: planning_progress: PlanningProgressReporter -@dataclass(frozen=True) -class TestExecutionPreparation: - """Prepared nested progress and execution reporters for test runs.""" - - progress: NestedCommandProgressCallbacks - execution_connection_progress: ConnectionProgressReporter - preflight_progress: TransientStatusReporter - effective_concurrency: int - worker_count: int - - @dataclass(frozen=True) class RulesCommandRequest: """Inputs for focused rule execution, inspection, or skill generation.""" diff --git a/src/sqlbuild/compiler/compile/main/effective_target_namespace.py b/src/sqlbuild/compiler/compile/main/_effective_target_namespace.py similarity index 100% rename from src/sqlbuild/compiler/compile/main/effective_target_namespace.py rename to src/sqlbuild/compiler/compile/main/_effective_target_namespace.py diff --git a/src/sqlbuild/compiler/discovery/main/discover_configuration.py b/src/sqlbuild/compiler/discovery/main/_discover_configuration.py similarity index 100% rename from src/sqlbuild/compiler/discovery/main/discover_configuration.py rename to src/sqlbuild/compiler/discovery/main/_discover_configuration.py diff --git a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py b/src/sqlbuild/compiler/migrations/_helpers/column_sql.py index f70055039d..4cacd717c9 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/column_sql.py @@ -5,6 +5,7 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import render_event_exists_sql, render_event_insert_sql from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -12,8 +13,6 @@ from sqlbuild.compiler.migrations._helpers.sql import ( decode_event_timestamp, optional_text, - render_event_exists_sql, - render_event_insert_sql, ) from sqlbuild.compiler.migrations.constants import ( COLUMN_MIGRATION_COLUMN_TYPES, diff --git a/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py b/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py index f14967b3a1..0f9e9b4887 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py @@ -6,6 +6,7 @@ from collections.abc import Callable, Iterable from typing import Any +from sqlbuild._native import render_event_exists_sql, render_event_insert_sql from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -13,8 +14,6 @@ from sqlbuild.compiler.migrations._helpers.sql import ( decode_event_timestamp, optional_text, - render_event_exists_sql, - render_event_insert_sql, ) from sqlbuild.compiler.migrations.constants import ( OLD_NAME_VIEW_COLUMN_TYPES, diff --git a/src/sqlbuild/compiler/migrations/_helpers/sql.py b/src/sqlbuild/compiler/migrations/_helpers/sql.py index 9773f4a003..0cf1616b6d 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/sql.py @@ -2,10 +2,11 @@ from __future__ import annotations -from collections.abc import Callable, Mapping +from collections.abc import Callable from datetime import UTC, datetime from typing import Any +from sqlbuild._native import render_event_exists_sql, render_event_insert_sql from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -18,8 +19,6 @@ from sqlbuild.compiler.migrations.exceptions import MigrationStateError from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal -from sqlbuild.sql_values.types import StateSqlValueType def qualified_migration_table( @@ -70,31 +69,6 @@ def build_insert_sql( ) -def render_event_exists_sql(*, table: str, event_id: str) -> str: - """Render the lookup that makes an append-only event write idempotent.""" - - event_id_literal: str = render_state_sql_literal( - value=event_id, declared_type=StateSqlValueType.STRING - ) - return f"SELECT event_id FROM {table} WHERE event_id = {event_id_literal}" - - -def render_event_insert_sql( - *, - table: str, - columns: tuple[str, ...], - column_types: Mapping[str, StateSqlValueType], - values: tuple[object | None, ...], -) -> str: - """Render one append-only event row as INSERT ... VALUES with typed literals.""" - - literals: str = ", ".join( - render_state_sql_literal(value=value, declared_type=column_types[column]) - for column, value in zip(columns, values, strict=True) - ) - return f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({literals})" - - def _event_table(*, event: MigrationEvent, render_qualified_name: Callable[..., str | None]) -> str: if event.destination.schema is None: raise MigrationStateError("model migration events require a destination schema") diff --git a/src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py b/src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py deleted file mode 100644 index 9f7882154b..0000000000 --- a/src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Build stable identifiers for immutable column migration events.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.compiler.migrations.models import MigrationRelation - - -def deterministic_column_migration_event_id( - *, - run_id: str, - target_name: str | None, - relation: MigrationRelation, - origin_column: str, - destination_column: str, -) -> str: - """Return the content-addressed ID for one column rename performed by one run.""" - - identity: tuple[object, ...] = ( - "column", - run_id, - target_name, - relation.identity, - origin_column.lower(), - destination_column.lower(), - ) - encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py b/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py deleted file mode 100644 index 7c9ecf2f68..0000000000 --- a/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Build stable identifiers for immutable model migration events.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.compiler.migrations.models import MigrationRelation - - -def deterministic_migration_event_id( - *, - run_id: str, - target_name: str | None, - origin: MigrationRelation, - destination: MigrationRelation, -) -> str: - """Return the content-addressed ID for one migration performed by one run.""" - - identity: tuple[object, ...] = ( - run_id, - target_name, - origin.identity, - destination.identity, - ) - encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/compiler/migrations/main/deterministic_old_name_view_event_id.py b/src/sqlbuild/compiler/migrations/main/deterministic_old_name_view_event_id.py deleted file mode 100644 index c1e36b912d..0000000000 --- a/src/sqlbuild/compiler/migrations/main/deterministic_old_name_view_event_id.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Build stable identifiers for immutable old-name view facts.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.compiler.migrations.types import OldNameViewEventType - - -def deterministic_old_name_view_event_id( - *, event_type: OldNameViewEventType, migration_event_id: str -) -> str: - """Return the content-addressed ID of one old-name step of one recorded move.""" - - identity: tuple[object, ...] = ("old_name", event_type.value, migration_event_id) - encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/compiler/pipeline/main/clone.py b/src/sqlbuild/compiler/pipeline/main/_clone.py similarity index 100% rename from src/sqlbuild/compiler/pipeline/main/clone.py rename to src/sqlbuild/compiler/pipeline/main/_clone.py diff --git a/src/sqlbuild/compiler/planner/main/commands/relation_plan.py b/src/sqlbuild/compiler/planner/main/commands/_relation_plan.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/commands/relation_plan.py rename to src/sqlbuild/compiler/planner/main/commands/_relation_plan.py diff --git a/src/sqlbuild/compiler/planner/main/commands/seed.py b/src/sqlbuild/compiler/planner/main/commands/_seed.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/commands/seed.py rename to src/sqlbuild/compiler/planner/main/commands/_seed.py diff --git a/src/sqlbuild/compiler/planner/main/commands/seed_state.py b/src/sqlbuild/compiler/planner/main/commands/_seed_state.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/commands/seed_state.py rename to src/sqlbuild/compiler/planner/main/commands/_seed_state.py diff --git a/src/sqlbuild/compiler/planner/main/commands/sql_test.py b/src/sqlbuild/compiler/planner/main/commands/_sql_test.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/commands/sql_test.py rename to src/sqlbuild/compiler/planner/main/commands/_sql_test.py diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/age_policy.py b/src/sqlbuild/compiler/source_freshness/_helpers/age_policy.py deleted file mode 100644 index 91bcab43e4..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/age_policy.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Source freshness age policy evaluation helpers.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild.compiler.source_freshness._helpers.datetime import require_aware_utc_datetime -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessInputError -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus -from sqlbuild.cursor_algebra.constants import MINUTE_TO_DAY_DURATION_UNITS -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy - - -def evaluate_source_freshness_age_policy( - *, - policy: SourceFreshnessAgePolicy | None, - data_version: object, - observed_at: datetime, -) -> SourceFreshnessAgeStatus | None: - """Evaluate optional timestamp age policy for one source freshness observation.""" - - if policy is None: - return None - if not isinstance(data_version, datetime): - return SourceFreshnessAgeStatus.UNKNOWN - aware_data_version: datetime = require_aware_utc_datetime( - value=data_version, field_name="data_version" - ) - aware_observed_at: datetime = require_aware_utc_datetime( - value=observed_at, field_name="observed_at" - ) - age_seconds: float = (aware_observed_at - aware_data_version).total_seconds() - if policy.error_after is not None and age_seconds > _duration_seconds(policy.error_after): - return SourceFreshnessAgeStatus.ERROR - if policy.warn_after is not None and age_seconds > _duration_seconds(policy.warn_after): - return SourceFreshnessAgeStatus.WARN - return SourceFreshnessAgeStatus.PASS - - -def _duration_seconds(value: str) -> int: - duration: Duration | None = Duration.parse(value) - if duration is None or not duration.is_single_unit_in(MINUTE_TO_DAY_DURATION_UNITS): - raise SourceFreshnessInputError( - "source freshness age_policy value must be a positive duration like 15m, 2h, or 1d" - ) - return duration.fixed_seconds diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/planning.py b/src/sqlbuild/compiler/source_freshness/_helpers/planning.py index eb530d0af4..3a29acfe20 100644 --- a/src/sqlbuild/compiler/source_freshness/_helpers/planning.py +++ b/src/sqlbuild/compiler/source_freshness/_helpers/planning.py @@ -6,11 +6,14 @@ from datetime import datetime from typing import Any -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.compiler.source_freshness._helpers.age_policy import ( +from sqlbuild._native import ( evaluate_source_freshness_age_policy, + normalize_source_freshness_data_version, + source_freshness_data_version_hash, + source_freshness_records_equivalent, ) +from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter +from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.compiler.source_freshness.constants import ( INCOMPLETE_CONFIGURATION_ERROR_FRAGMENT, PHYSICAL_TABLE_SOURCE_ERROR_FRAGMENT, @@ -19,19 +22,10 @@ from sqlbuild.compiler.source_freshness.main.adapter_observation import ( observe_adapter_sources_freshness, ) -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) from sqlbuild.compiler.source_freshness.main.observation import ( observe_configured_source_freshness, ) from sqlbuild.compiler.source_freshness.main.read import read_latest_source_freshness -from sqlbuild.compiler.source_freshness.main.record_equivalence import ( - source_freshness_records_equivalent, -) from sqlbuild.compiler.source_freshness.models import ( AdapterSourceFreshnessBatch, DirectSourceFreshnessPlanningResult, diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/state.py b/src/sqlbuild/compiler/source_freshness/_helpers/state.py deleted file mode 100644 index c3e559a007..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/state.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Shared source freshness state normalization and hashing helpers.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import UTC, datetime, timedelta - -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.types import SourceFreshnessComparableRecord -from sqlbuild.cursor_algebra.constants import MINUTE_TO_DAY_DURATION_UNITS -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind - - -def normalize_source_freshness_data_version( - *, value: object, value_kind: SourceFreshnessValueKind -) -> str: - """Normalize a source freshness value for stable state storage.""" - - match value_kind: - case SourceFreshnessValueKind.TIMESTAMP: - if not isinstance(value, datetime): - raise SourceFreshnessObservationError( - "Timestamp source freshness values must be datetime values" - ) - normalized: datetime = value - if normalized.tzinfo is not None: - normalized = normalized.astimezone(UTC) - return normalized.isoformat() - case SourceFreshnessValueKind.INTEGER: - if isinstance(value, bool) or not isinstance(value, int): - raise SourceFreshnessObservationError( - "Integer source freshness values must be integer values" - ) - return str(value) - case SourceFreshnessValueKind.STRING: - if not isinstance(value, str): - raise SourceFreshnessObservationError( - "String source freshness values must be string values" - ) - return value - raise SourceFreshnessObservationError(f"Unsupported source freshness value kind: {value_kind}") - - -def source_freshness_data_version_hash( - *, - source_name: str, - strategy: SourceFreshnessStrategy | str, - value_kind: SourceFreshnessValueKind | str, - data_version: str, -) -> str: - """Hash the stable source freshness identity used as a graph input.""" - - payload: dict[str, str] = { - "source_name": source_name, - "strategy": strategy.value if isinstance(strategy, SourceFreshnessStrategy) else strategy, - "value_kind": value_kind.value - if isinstance(value_kind, SourceFreshnessValueKind) - else value_kind, - "data_version": data_version, - } - encoded: bytes = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() - return hashlib.sha256(encoded).hexdigest() - - -def source_freshness_records_equivalent( - *, - previous_record: SourceFreshnessComparableRecord, - current_record: SourceFreshnessComparableRecord, - lag_tolerance: str | None = None, -) -> bool: - """Return whether two source freshness records are equivalent for skip decisions.""" - - if previous_record.data_version_hash == current_record.data_version_hash: - return True - if previous_record.value_kind != SourceFreshnessValueKind.TIMESTAMP.value: - return False - if current_record.value_kind != SourceFreshnessValueKind.TIMESTAMP.value: - return False - if previous_record.data_version is None or current_record.data_version is None: - return False - previous_timestamp: datetime = _parse_timestamp_data_version(previous_record.data_version) - current_timestamp: datetime = _parse_timestamp_data_version(current_record.data_version) - if current_timestamp == previous_timestamp: - return True - if lag_tolerance is None: - return False - if current_timestamp < previous_timestamp: - return False - return current_timestamp - previous_timestamp <= _parse_lag_tolerance(lag_tolerance) - - -def _parse_timestamp_data_version(value: str) -> datetime: - try: - parsed: datetime = datetime.fromisoformat(value) - except ValueError as exc: - raise SourceFreshnessObservationError( - f"Timestamp source freshness state value is not valid ISO datetime: {value}" - ) from exc - if parsed.tzinfo is None or parsed.utcoffset() is None: - return parsed.replace(tzinfo=UTC) - return parsed.astimezone(UTC) - - -def _parse_lag_tolerance(value: str) -> timedelta: - duration: Duration | None = Duration.parse(value) - if duration is None or not duration.is_single_unit_in(MINUTE_TO_DAY_DURATION_UNITS): - raise SourceFreshnessObservationError( - "source freshness lag_tolerance must be a positive duration like 15m, 2h, or 1d" - ) - return timedelta(seconds=duration.fixed_seconds) diff --git a/src/sqlbuild/compiler/source_freshness/main/age_policy.py b/src/sqlbuild/compiler/source_freshness/main/age_policy.py deleted file mode 100644 index b353fbfe85..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/age_policy.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public source freshness age policy evaluation entrypoint.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild.compiler.source_freshness._helpers.age_policy import ( - evaluate_source_freshness_age_policy as _evaluate_source_freshness_age_policy, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus -from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy - - -def evaluate_source_freshness_age_policy( - *, - policy: SourceFreshnessAgePolicy | None, - data_version: object, - observed_at: datetime, -) -> SourceFreshnessAgeStatus | None: - """Evaluate optional timestamp age policy for one source freshness observation.""" - - return _evaluate_source_freshness_age_policy( - policy=policy, - data_version=data_version, - observed_at=observed_at, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/data_version_hash.py b/src/sqlbuild/compiler/source_freshness/main/data_version_hash.py deleted file mode 100644 index e291e8245d..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/data_version_hash.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public shared source freshness data-version hash entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.source_freshness._helpers.state import ( - source_freshness_data_version_hash as _source_freshness_data_version_hash, -) -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind - - -def source_freshness_data_version_hash( - *, - source_name: str, - strategy: SourceFreshnessStrategy | str, - value_kind: SourceFreshnessValueKind | str, - data_version: str, -) -> str: - """Hash the stable source freshness identity used as a graph input.""" - - return _source_freshness_data_version_hash( - source_name=source_name, - strategy=strategy, - value_kind=value_kind, - data_version=data_version, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/normalization.py b/src/sqlbuild/compiler/source_freshness/main/normalization.py deleted file mode 100644 index 338f95aeb5..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/normalization.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public shared source freshness normalization entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.source_freshness._helpers.state import ( - normalize_source_freshness_data_version as _normalize_source_freshness_data_version, -) -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind - - -def normalize_source_freshness_data_version( - *, value: object, value_kind: SourceFreshnessValueKind -) -> str: - """Normalize a source freshness value for stable state storage.""" - - return _normalize_source_freshness_data_version(value=value, value_kind=value_kind) diff --git a/src/sqlbuild/compiler/source_freshness/main/record_equivalence.py b/src/sqlbuild/compiler/source_freshness/main/record_equivalence.py deleted file mode 100644 index 35bbd2a6f1..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/record_equivalence.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public source freshness record equivalence entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.source_freshness._helpers.state import ( - source_freshness_records_equivalent as _source_freshness_records_equivalent, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessComparableRecord - - -def source_freshness_records_equivalent( - *, - previous_record: SourceFreshnessComparableRecord, - current_record: SourceFreshnessComparableRecord, - lag_tolerance: str | None = None, -) -> bool: - """Return whether two source freshness records are equivalent for skip decisions.""" - - return _source_freshness_records_equivalent( - previous_record=previous_record, - current_record=current_record, - lag_tolerance=lag_tolerance, - ) diff --git a/src/sqlbuild/diagnostics/_helpers/redaction.py b/src/sqlbuild/diagnostics/_helpers/redaction.py index 2a522b3164..169d967780 100644 --- a/src/sqlbuild/diagnostics/_helpers/redaction.py +++ b/src/sqlbuild/diagnostics/_helpers/redaction.py @@ -1,79 +1,21 @@ -"""Secret-safe diagnostic record rendering.""" +"""Redacted structured fields of one logging record.""" from __future__ import annotations import logging -import re -from collections.abc import Mapping from typing import Final +from sqlbuild._native import redact_diagnostic_value from sqlbuild.diagnostics.constants import FORMATTER_RECORD_FIELDS, SQL_TEXT_FIELD -_REDACTED: Final[str] = "[REDACTED]" -_SECRET_FIELD_PARTS: Final[tuple[str, ...]] = ( - "access_key", - "api_key", - "authorization", - "client_secret", - "credential", - "dsn", - "password", - "private_key", - "refresh_token", - "secret", - "token", -) -_SECRET_VALUE_PATTERN: Final[re.Pattern[str]] = re.compile( - r"(?i)\b(api[-_]?key|access[-_]?key|private[-_]?key|authorization|" - r"client[-_]?secret|refresh[-_]?token|" - r"password|passwd|pwd|secret|token|dsn|connection_string)" - r"(\s*[=:]\s*)(\"[^\"\r\n]{0,512}\"|'[^'\r\n]{0,512}'|[^\s,;]{1,512})" -) -_AUTHORIZATION_PATTERN: Final[re.Pattern[str]] = re.compile( - r"(?i)\b(authorization)(\s*[=:]\s*)(?:basic|bearer)\s+" - r"(\"[^\"\r\n]{1,512}\"|'[^'\r\n]{1,512}'|[^\s,;]{1,512})" -) -_BEARER_PATTERN: Final[re.Pattern[str]] = re.compile(r"(?i)\bbearer\s+[a-z0-9._~+/=-]{1,512}") -_URI_CREDENTIAL_PATTERN: Final[re.Pattern[str]] = re.compile( - r"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@]+:)([^\s@]+)(@)" -) _STANDARD_RECORD_FIELDS: Final[frozenset[str]] = frozenset(logging.makeLogRecord({}).__dict__) -def redact_text(value: str) -> str: - """Redact obvious named secrets and URI passwords from diagnostic text.""" - - authorization_redacted: str = _AUTHORIZATION_PATTERN.sub(rf"\1\2{_REDACTED}", value) - bearer_redacted: str = _BEARER_PATTERN.sub(f"Bearer {_REDACTED}", authorization_redacted) - named_redacted: str = _SECRET_VALUE_PATTERN.sub(rf"\1\2{_REDACTED}", bearer_redacted) - return _URI_CREDENTIAL_PATTERN.sub(rf"\1{_REDACTED}\3", named_redacted) - - -def redact_value(*, name: str, value: object) -> object: - """Return a persistence-safe value for one structured logging field.""" - - lowered_name: str = name.lower().replace("-", "_") - if any(part in lowered_name for part in _SECRET_FIELD_PARTS): - return _REDACTED - if isinstance(value, str): - return redact_text(value) - if value is None or isinstance(value, (bool, int, float)): - return value - if isinstance(value, Mapping): - return {str(key): redact_value(name=str(key), value=item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [redact_value(name=name, value=item) for item in value] - try: - return redact_text(str(value)) - except Exception: - return f"<{type(value).__name__}>" - - def safe_extra_fields(record: logging.LogRecord) -> dict[str, object]: """Collect redacted non-standard fields without evaluating arbitrary formatters.""" return { - key: redact_value(name=key, value=value) + key: redact_diagnostic_value(name=key, value=value) for key, value in record.__dict__.items() if key not in _STANDARD_RECORD_FIELDS and key not in FORMATTER_RECORD_FIELDS diff --git a/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py b/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py index 026d7ea792..d7a4a12f56 100644 --- a/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py +++ b/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py @@ -4,7 +4,8 @@ import logging -from sqlbuild.diagnostics._helpers.redaction import redact_text, redact_value, safe_extra_fields +from sqlbuild._native import redact_diagnostic_text, redact_diagnostic_value +from sqlbuild.diagnostics._helpers.redaction import safe_extra_fields class DiagnosticRecordRedactor: @@ -14,13 +15,13 @@ class DiagnosticRecordRedactor: def text(value: str) -> str: """Redact obvious secrets from diagnostic text.""" - return redact_text(value) + return redact_diagnostic_text(value) @staticmethod def value(*, name: str, value: object) -> object: """Redact one structured field value.""" - return redact_value(name=name, value=value) + return redact_diagnostic_value(name=name, value=value) @staticmethod def extras(record: logging.LogRecord) -> dict[str, object]: diff --git a/src/sqlbuild/executor/diff/constants.py b/src/sqlbuild/executor/diff/constants.py index 320cbd10ed..5387092604 100644 --- a/src/sqlbuild/executor/diff/constants.py +++ b/src/sqlbuild/executor/diff/constants.py @@ -4,8 +4,6 @@ ROW_DIFF_RELATIVE_TOLERANCE_KEY: str = "relative" DIFF_INPUT_KIND_MODEL: str = "model" DIFF_INPUT_KIND_QUERY: str = "query" -ROW_DIFF_KEY_ERROR_PREFIX: str = "row diff " -ROW_DIFF_KEY_ERROR_FRAGMENT: str = " unique_key values" ROW_DIFF_TOLERANCE_KEYS: frozenset[str] = frozenset( (ROW_DIFF_ABSOLUTE_TOLERANCE_KEY, ROW_DIFF_RELATIVE_TOLERANCE_KEY) ) diff --git a/src/sqlbuild/executor/diff/main/config.py b/src/sqlbuild/executor/diff/main/_config.py similarity index 100% rename from src/sqlbuild/executor/diff/main/config.py rename to src/sqlbuild/executor/diff/main/_config.py diff --git a/src/sqlbuild/executor/diff/main/execute_query.py b/src/sqlbuild/executor/diff/main/_execute_query.py similarity index 100% rename from src/sqlbuild/executor/diff/main/execute_query.py rename to src/sqlbuild/executor/diff/main/_execute_query.py diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py index ab0bb19620..31231a63b4 100644 --- a/src/sqlbuild/executor/migrations/_helpers/columns.py +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -5,13 +5,11 @@ from datetime import UTC, datetime from typing import Any +from sqlbuild._native import deterministic_column_migration_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.deterministic_column_event_id import ( - deterministic_column_migration_event_id, -) from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.main.write_column_event import write_column_migration_event from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py index 685e755c6c..2ddafcdb9f 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py @@ -5,11 +5,9 @@ from datetime import datetime from typing import Any +from sqlbuild._native import deterministic_old_name_view_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.deterministic_old_name_view_event_id import ( - deterministic_old_name_view_event_id, -) from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( read_old_name_view_events, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index 9603e3bb2e..45656d83b5 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -8,6 +8,7 @@ from datetime import UTC, datetime, timedelta from typing import Any +from sqlbuild._native import deterministic_migration_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.main.grants_for_columns import grants_for_columns @@ -19,9 +20,6 @@ from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME -from sqlbuild.compiler.migrations.main.deterministic_event_id import ( - deterministic_migration_event_id, -) from sqlbuild.compiler.migrations.main.read_recorded_column_renames import ( read_recorded_column_renames, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py index e028be67bd..e3cd213f9c 100644 --- a/src/sqlbuild/executor/migrations/_helpers/promotion.py +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -5,12 +5,10 @@ from datetime import UTC, datetime from typing import Any +from sqlbuild._native import deterministic_migration_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.deterministic_event_id import ( - deterministic_migration_event_id, -) from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation, OldNameViewEvent diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index dcce6ff0dd..e33c45671b 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime from typing import Any +from sqlbuild._native import deterministic_old_name_view_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo @@ -19,9 +20,6 @@ MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, ) -from sqlbuild.compiler.migrations.main.deterministic_old_name_view_event_id import ( - deterministic_old_name_view_event_id, -) from sqlbuild.compiler.migrations.main.read_old_name_view_histories import ( read_old_name_view_histories, ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/graph/core.py b/src/sqlbuild/integrations/dbt/_helpers/graph/core.py index 80290e1baf..3a6ff7d900 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/graph/core.py +++ b/src/sqlbuild/integrations/dbt/_helpers/graph/core.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild._native import invert_edges, transitive_closure +from sqlbuild._native import invert_edges, resolve_sqlbuild_model_dbt_refs, transitive_closure from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) @@ -11,9 +11,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( - resolve_sqlbuild_model_dbt_refs, -) from sqlbuild.integrations.dbt.models import ( DbtCombinedGraph, DbtCombinedGraphKey, diff --git a/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py b/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py deleted file mode 100644 index 77d0698d31..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Compile-time dbt ref helpers used by SQLBuild model attachment.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.integrations.dbt._helpers.manifest.core import ( - build_dbt_manifest_index, - resolve_dbt_manifest_model, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def build_compile_dbt_manifest_index(*, manifest_contents: str | None) -> DbtManifestIndex | None: - """Build a dbt manifest index from discovered manifest contents.""" - - if manifest_contents is None: - return None - try: - raw_data: object = json.loads(manifest_contents) - except json.JSONDecodeError as exc: - raise CompileInputError( - f"Invalid dbt manifest JSON: {exc.msg}", - code="C211", - ) from exc - return build_dbt_manifest_index(raw_data=raw_data) - - -def validate_compile_dbt_model_names( - *, known_model_names: set[str], dbt_manifest: DbtManifestIndex | None -) -> None: - """Reject ambiguous ownership between dbt and SQLBuild models.""" - - if dbt_manifest is None: - return - duplicate_names: tuple[str, ...] = tuple( - sorted(name for name in known_model_names if name in dbt_manifest.models_by_name) - ) - if duplicate_names: - raise CompileInputError( - f"dbt and SQLBuild models share names: {', '.join(duplicate_names)}", - code="C215", - help=( - "Rename either the dbt model or SQLBuild model; owner-qualified names " - "are not supported." - ), - ) - - -def validate_compile_dbt_model_reference( - *, - reference: CompileSqlReference, - model_relative_path: Path, - dbt_manifest: DbtManifestIndex | None, -) -> None: - """Validate one compile-time dbt ref against a dbt manifest.""" - - if reference.ref_kind != SqlReferenceKind.DBT_REF: - return - if dbt_manifest is None: - raise CompileInputError( - f"Model file {model_relative_path} uses " - f"{SqlReferenceKind.DBT_REF.example_call(reference.ref_name, quote='"')} but no dbt " - "manifest was found", - code="C214", - help=( - "Run dbt compile or configure dbt target_path so SQLBuild can read manifest.json." - ), - ) - resolve_dbt_manifest_model( - manifest=dbt_manifest, - package_name=reference.ref_package, - name=reference.ref_name, - ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/manifest/core.py b/src/sqlbuild/integrations/dbt/_helpers/manifest/core.py deleted file mode 100644 index e0bde28e89..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/manifest/core.py +++ /dev/null @@ -1,312 +0,0 @@ -"""dbt manifest loading and model lookup helpers.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, cast - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.integrations.dbt.constants import ( - DBT_MANIFEST_CONFIG_KEY, - DBT_MANIFEST_MATERIALIZED_KEY, -) -from sqlbuild.integrations.dbt.models import ( - DbtManifestIndex, - DbtManifestModel, - DbtManifestSeed, - DbtManifestSource, -) -from sqlbuild.integrations.dbt.types import DbtSupportedResourceType - -_INDEXED_NODE_RESOURCE_TYPES: frozenset[DbtSupportedResourceType] = frozenset( - {DbtSupportedResourceType.MODEL, DbtSupportedResourceType.SNAPSHOT} -) - - -def load_dbt_manifest_index(*, manifest_path: Path) -> DbtManifestIndex: - """Load and index dbt model nodes from a manifest.json file.""" - - if not manifest_path.is_file(): - raise CompileInputError( - f"dbt manifest file does not exist: {manifest_path}", - code="C210", - help="Run dbt compile or configure dbt target_path so SQLBuild can read manifest.json.", - ) - try: - raw_data: Any = json.loads(manifest_path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise CompileInputError( - f"Invalid dbt manifest JSON: {exc.msg}", - code="C211", - ) from exc - return build_dbt_manifest_index(raw_data=raw_data) - - -def build_dbt_manifest_index(*, raw_data: object) -> DbtManifestIndex: - """Build dbt model lookup indexes from decoded manifest JSON.""" - - if not isinstance(raw_data, dict): - raise CompileInputError("Invalid dbt manifest: root must be an object", code="C211") - manifest_data: dict[str, object] = cast(dict[str, object], raw_data) - raw_nodes: object | None = manifest_data.get("nodes") - if not isinstance(raw_nodes, dict): - raise CompileInputError("Invalid dbt manifest: nodes must be an object", code="C211") - raw_sources: object | None = manifest_data.get("sources") - if raw_sources is None: - raw_sources = {} - if not isinstance(raw_sources, dict): - raise CompileInputError("Invalid dbt manifest: sources must be an object", code="C211") - raw_unit_tests: object | None = manifest_data.get("unit_tests") - if raw_unit_tests is None: - raw_unit_tests = {} - if not isinstance(raw_unit_tests, dict): - raise CompileInputError("Invalid dbt manifest: unit_tests must be an object", code="C211") - - models_by_unique_id: dict[str, DbtManifestModel] = {} - models_by_name_lists: dict[str, list[DbtManifestModel]] = {} - models_by_package_and_name: dict[tuple[str, str], DbtManifestModel] = {} - sources_by_unique_id: dict[str, DbtManifestSource] = {} - seeds_by_unique_id: dict[str, DbtManifestSeed] = {} - validation_depends_on_nodes_by_unique_id: dict[str, tuple[str, ...]] = {} - unique_id: object - raw_node: object - for unique_id, raw_node in raw_nodes.items(): - if not isinstance(unique_id, str) or not isinstance(raw_node, dict): - continue - node_data: dict[object, object] = cast(dict[object, object], raw_node) - resource_type: object = node_data.get("resource_type") - if resource_type in _INDEXED_NODE_RESOURCE_TYPES: - model: DbtManifestModel = _parse_model(unique_id=unique_id, raw_node=node_data) - models_by_unique_id[model.unique_id] = model - models_by_name_lists.setdefault(model.name, []).append(model) - models_by_package_and_name[(model.package_name, model.name)] = model - continue - if resource_type == DbtSupportedResourceType.SEED: - seed: DbtManifestSeed = _parse_seed(unique_id=unique_id, raw_node=node_data) - seeds_by_unique_id[seed.unique_id] = seed - continue - raw_validation_nodes: dict[object, object] - for raw_validation_nodes in ( - cast(dict[object, object], raw_nodes), - cast(dict[object, object], raw_unit_tests), - ): - for unique_id, raw_node in raw_validation_nodes.items(): - if not isinstance(unique_id, str) or not isinstance(raw_node, dict): - continue - node_data = cast(dict[object, object], raw_node) - if node_data.get("resource_type") not in ( - DbtSupportedResourceType.TEST, - DbtSupportedResourceType.UNIT_TEST, - ): - continue - validation_depends_on_nodes_by_unique_id[unique_id] = _parse_depends_on_nodes( - node_data.get("depends_on") - ) - - for unique_id, raw_node in raw_sources.items(): - if not isinstance(unique_id, str) or not isinstance(raw_node, dict): - continue - node_data = cast(dict[object, object], raw_node) - if node_data.get("resource_type") != DbtSupportedResourceType.SOURCE: - continue - source: DbtManifestSource = _parse_source(unique_id=unique_id, raw_node=node_data) - sources_by_unique_id[source.unique_id] = source - - return DbtManifestIndex( - models_by_unique_id=models_by_unique_id, - models_by_name={name: tuple(models) for name, models in models_by_name_lists.items()}, - models_by_package_and_name=models_by_package_and_name, - sources_by_unique_id=sources_by_unique_id, - seeds_by_unique_id=seeds_by_unique_id, - validation_depends_on_nodes_by_unique_id=validation_depends_on_nodes_by_unique_id, - ) - - -def resolve_dbt_manifest_model( - *, manifest: DbtManifestIndex, name: str, package_name: str | None = None -) -> DbtManifestModel: - """Resolve a dbt model by one-arg or package-qualified ref semantics.""" - - if package_name is not None: - model: DbtManifestModel | None = manifest.models_by_package_and_name.get( - (package_name, name) - ) - if model is None: - raise CompileInputError( - f"dbt model '{package_name}.{name}' was not found in manifest", - code="C212", - help=( - f"Check the {SqlReferenceKind.DBT_REF.placeholder_call('...')} package/name " - "or run dbt compile to refresh manifest.json." - ), - ) - return model - - matches: tuple[DbtManifestModel, ...] = manifest.models_by_name.get(name, ()) - if not matches: - raise CompileInputError( - f"dbt model '{name}' was not found in manifest", - code="C212", - help=( - f"Check the {SqlReferenceKind.DBT_REF.placeholder_call('...')} name or run dbt " - "compile to refresh manifest.json." - ), - ) - if len(matches) > 1: - packages: str = ", ".join(sorted(model.package_name for model in matches)) - raise CompileInputError( - f"dbt model '{name}' is ambiguous across packages: {packages}", - code="C213", - help=( - f"Use {SqlReferenceKind.DBT_REF.example_call('package_name', name, quote='"')} to " - "choose one dbt model." - ), - ) - return matches[0] - - -def dbt_manifest_model_materialization(*, model: DbtManifestModel) -> str | None: - """Return the lowercased dbt materialization for a manifest model, if declared.""" - - config: object | None = model.payload.get(DBT_MANIFEST_CONFIG_KEY) - if not isinstance(config, dict): - return None - materialized: object | None = cast(dict[str, object], config).get(DBT_MANIFEST_MATERIALIZED_KEY) - if not isinstance(materialized, str) or not materialized.strip(): - return None - return materialized.strip().lower() - - -def _parse_model(*, unique_id: str, raw_node: dict[object, object]) -> DbtManifestModel: - package_name: str = _required_str(value=raw_node.get("package_name"), field_name="package_name") - name: str = _required_str(value=raw_node.get("name"), field_name="name") - database: str | None = _optional_str(raw_node.get("database")) - schema: str | None = _optional_str(raw_node.get("schema")) - alias: str | None = _optional_str(raw_node.get("alias")) - relation_name: str | None = _optional_str(raw_node.get("relation_name")) - if relation_name is None: - relation_name = _render_relation_name(database=database, schema=schema, name=alias or name) - depends_on_nodes: tuple[str, ...] = _parse_depends_on_nodes(raw_node.get("depends_on")) - return DbtManifestModel( - unique_id=unique_id, - package_name=package_name, - name=name, - database=database, - schema=schema, - alias=alias, - node_checksum=_parse_checksum(raw_node.get("checksum")), - relation_name=relation_name, - fqn=_parse_fqn(raw_node.get("fqn")), - query_sql=_model_query_sql(raw_node=raw_node), - depends_on_nodes=depends_on_nodes, - payload={str(key): value for key, value in raw_node.items()}, - ) - - -def _parse_source(*, unique_id: str, raw_node: dict[object, object]) -> DbtManifestSource: - package_name: str = _required_str(value=raw_node.get("package_name"), field_name="package_name") - source_name: str = _required_str(value=raw_node.get("source_name"), field_name="source_name") - name: str = _required_str(value=raw_node.get("name"), field_name="name") - database: str | None = _optional_str(raw_node.get("database")) - schema: str | None = _optional_str(raw_node.get("schema")) - identifier: str | None = _optional_str(raw_node.get("identifier")) - relation_name: str | None = _optional_str(raw_node.get("relation_name")) - if relation_name is None: - relation_name = _render_relation_name( - database=database, - schema=schema, - name=identifier or name, - ) - return DbtManifestSource( - unique_id=unique_id, - package_name=package_name, - source_name=source_name, - name=name, - database=database, - schema=schema, - identifier=identifier, - relation_name=relation_name, - payload={str(key): value for key, value in raw_node.items()}, - ) - - -def _parse_seed(*, unique_id: str, raw_node: dict[object, object]) -> DbtManifestSeed: - package_name: str = _required_str(value=raw_node.get("package_name"), field_name="package_name") - name: str = _required_str(value=raw_node.get("name"), field_name="name") - database: str | None = _optional_str(raw_node.get("database")) - schema: str | None = _optional_str(raw_node.get("schema")) - alias: str | None = _optional_str(raw_node.get("alias")) - relation_name: str | None = _optional_str(raw_node.get("relation_name")) - if relation_name is None: - relation_name = _render_relation_name(database=database, schema=schema, name=alias or name) - return DbtManifestSeed( - unique_id=unique_id, - package_name=package_name, - name=name, - database=database, - schema=schema, - alias=alias, - relation_name=relation_name, - payload={str(key): value for key, value in raw_node.items()}, - ) - - -def _render_relation_name(*, database: str | None, schema: str | None, name: str) -> str: - parts: tuple[str, ...] = tuple(part for part in (database, schema, name) if part) - return ".".join(parts) - - -def _model_query_sql(*, raw_node: dict[object, object]) -> str: - return ( - _optional_str(raw_node.get("raw_code")) - or _optional_str(raw_node.get("compiled_code")) - or _optional_str(raw_node.get("raw_sql")) - or _optional_str(raw_node.get("compiled_sql")) - or "" - ) - - -def _required_str(*, value: object, field_name: str) -> str: - if isinstance(value, str) and value: - return value - raise CompileInputError(f"Invalid dbt manifest: model {field_name} is required", code="C211") - - -def _optional_str(value: object) -> str | None: - if isinstance(value, str) and value: - return value - return None - - -def _parse_fqn(value: object) -> tuple[str, ...]: - if not isinstance(value, list): - return () - parts: list[str] = [] - item: object - for item in value: - if not isinstance(item, str) or not item: - return () - parts.append(item) - return tuple(parts) - - -def _parse_checksum(value: object) -> str | None: - if isinstance(value, str) and value: - return value - if not isinstance(value, dict): - return None - raw_checksum: dict[str, object] = cast(dict[str, object], value) - checksum: object | None = raw_checksum.get("checksum") - return checksum if isinstance(checksum, str) and checksum else None - - -def _parse_depends_on_nodes(value: object) -> tuple[str, ...]: - if not isinstance(value, dict): - return () - depends_on: dict[str, object] = cast(dict[str, object], value) - raw_nodes: object | None = depends_on.get("nodes") - if not isinstance(raw_nodes, list): - return () - return tuple(node for node in raw_nodes if isinstance(node, str) and node) diff --git a/src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py b/src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py deleted file mode 100644 index 3a63266d32..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Helpers for SQLBuild model references into dbt manifests.""" - -from __future__ import annotations - -from collections.abc import Sequence - -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.integrations.dbt._helpers.manifest.core import resolve_dbt_manifest_model -from sqlbuild.integrations.dbt.models import DbtManifestIndex, DbtManifestModel - - -def resolve_sqlbuild_model_dbt_refs( - *, - project: CompiledProjectView, - manifest: DbtManifestIndex, - selected_model_names: Sequence[str] | None = None, -) -> tuple[tuple[CompiledModelView, DbtManifestModel], ...]: - """Return resolved dbt refs used directly by SQLBuild models.""" - - selected_names: frozenset[str] | None = ( - None if selected_model_names is None else frozenset(selected_model_names) - ) - refs: list[tuple[CompiledModelView, DbtManifestModel]] = [] - model: CompiledModelView - for model in project.models: - if selected_names is not None and model.name not in selected_names: - continue - for reference in model.references: - if reference.ref_kind != SqlReferenceKind.DBT_REF: - continue - refs.append( - ( - model, - resolve_dbt_manifest_model( - manifest=manifest, - package_name=reference.ref_package, - name=reference.ref_name, - ), - ) - ) - return tuple(refs) - - -def resolve_dbt_reference_relation( - *, - manifest: DbtManifestIndex | None, - ref_kind: str, - ref_name: str, - ref_package: str | None, -) -> str | None: - """Resolve one external dbt reference to its manifest relation.""" - - if ref_kind != SqlReferenceKind.DBT_REF or manifest is None: - return None - return resolve_dbt_manifest_model( - manifest=manifest, - package_name=ref_package, - name=ref_name, - ).relation_name diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py index 9173ead186..1df7b8fb1f 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py @@ -4,7 +4,11 @@ from dataclasses import replace -from sqlbuild._native import ConnectionProgressReporter, apply_command_warehouse +from sqlbuild._native import ( + ConnectionProgressReporter, + apply_command_warehouse, + render_dbt_interop_plan, +) from sqlbuild.cli.commands.main.dbt.dbt_sqlbuild_work import execute_dbt_sqlbuild_work from sqlbuild.cli.commands.main.execution.connection_progress import ( build_connection_progress_reporter, @@ -18,7 +22,6 @@ render_dbt_execution_summary_footer, ) from sqlbuild.integrations.dbt._helpers.pipeline.plan_output import build_sqlbuild_plan_output -from sqlbuild.integrations.dbt.main.pipeline.render_plan import render_dbt_interop_plan from sqlbuild.integrations.dbt.main.profile._resolve_connection_config import ( resolve_connection_config, ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index d25ac65540..20c24aa6fd 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -8,7 +8,11 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + discover_project_inputs, + load_dbt_manifest_index, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -24,9 +28,6 @@ from sqlbuild.integrations.dbt.main.config._resolve_plan_options import resolve_dbt_plan_options from sqlbuild.integrations.dbt.main.config._resolve_vars_mapping import resolve_dbt_vars_mapping from sqlbuild.integrations.dbt.main.graph._build_combined_graph import build_dbt_combined_graph -from sqlbuild.integrations.dbt.main.manifest._load_manifest_index import ( - load_manifest_index as load_dbt_manifest_index, -) from sqlbuild.integrations.dbt.main.manifest._resolve_manifest_path import ( resolve_dbt_manifest_path, ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py index 350a8f07e8..f2d0be7060 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py +++ b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py @@ -4,16 +4,13 @@ from collections.abc import Sequence +from sqlbuild._native import build_dbt_interop_plan, resolve_sqlbuild_model_dbt_refs from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion from sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors import ( resolve_sqlbuild_model_selector_names, ) from sqlbuild.compiler.planner.models import SelectorExpansion -from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( - resolve_sqlbuild_model_dbt_refs, -) -from sqlbuild.integrations.dbt._helpers.planning.plan import build_dbt_interop_plan from sqlbuild.integrations.dbt._helpers.selection.core import resolve_dbt_interop_sqlbuild_selection from sqlbuild.integrations.dbt._helpers.selection.selector_terms import dbt_fqn_selector_term from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py b/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py deleted file mode 100644 index 3ab0487e19..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py +++ /dev/null @@ -1,249 +0,0 @@ -"""Plan construction and formatting for ordinary dbt interop commands.""" - -from __future__ import annotations - -import json -import re -from collections import defaultdict -from collections.abc import Sequence - -from sqlbuild._native import append_overflow_line, format_plan, visible_entries -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.integrations.dbt.models import DbtInteropPlan, DbtInteropSelectionResult, DbtLsNode -from sqlbuild.integrations.dbt.types import DbtInteropCommand, DbtInteropSkipReason -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.surface_header import format_surface_header -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.models import DisplayOptions - -_ANSI_ESCAPE_PATTERN: re.Pattern[str] = re.compile(r"\033\[[0-9;]*m") - - -def build_dbt_interop_plan( - *, - command: DbtInteropCommand | str, - dbt_command_argv: Sequence[str], - dbt_ls_nodes: Sequence[DbtLsNode], - sqlbuild_command_argvs: Sequence[Sequence[str]], - selection: DbtInteropSelectionResult, - dbt_required_selector_terms: Sequence[str] = (), - supplemental_dbt_command_argvs: Sequence[Sequence[str]] = (), - warnings: Sequence[str] = (), - sqlbuild_plan_output: PlanOutput | None = None, -) -> DbtInteropPlan: - """Build a display-ready selection plan without dbt model state.""" - - normalized_command: DbtInteropCommand = DbtInteropCommand(command) - dbt_selected_unique_ids: tuple[str, ...] = tuple( - sorted(node.unique_id for node in dbt_ls_nodes) - ) - dbt_has_work: bool = bool(dbt_selected_unique_ids or selection.dbt_required_unique_ids) - sqlbuild_has_work: bool = bool(selection.sqlbuild_model_names) - resolved_warnings: tuple[str, ...] = tuple(warnings) - no_match_warning: str = "No dbt or SQLBuild resources matched the selection." - if not dbt_has_work and not sqlbuild_has_work and no_match_warning not in resolved_warnings: - resolved_warnings = (*resolved_warnings, no_match_warning) - return DbtInteropPlan( - command=normalized_command, - dbt_command_argv=tuple(dbt_command_argv), - dbt_selected_nodes=tuple(dbt_ls_nodes), - dbt_selected_unique_ids=dbt_selected_unique_ids, - sqlbuild_command_argvs=tuple(tuple(argv) for argv in sqlbuild_command_argvs), - selection=selection, - sqlbuild_plan_output=sqlbuild_plan_output, - dbt_required_selector_terms=tuple(dbt_required_selector_terms), - supplemental_dbt_command_argvs=tuple( - tuple(argv) for argv in supplemental_dbt_command_argvs - ), - dbt_skip_reason=None if dbt_has_work else DbtInteropSkipReason.NO_DBT_WORK, - sqlbuild_skip_reason=None if sqlbuild_has_work else DbtInteropSkipReason.NO_SQLBUILD_WORK, - warnings=resolved_warnings, - ) - - -def format_dbt_interop_plan( - *, - plan: DbtInteropPlan, - use_color: bool = True, - display_options: DisplayOptions | None = None, -) -> str: - """Format selected dbt and SQLBuild work without change classifications.""" - - options: DisplayOptions = display_options or DisplayOptions() - selected_count: int = len( - frozenset((*plan.dbt_selected_unique_ids, *plan.selection.dbt_required_unique_ids)) - ) + len(plan.selection.sqlbuild_model_names) - style: CliStyle = CliStyle(use_color=True) - lines: list[str] = [ - format_surface_header( - style=style, title="Plan ready", context=f"{selected_count} selected resources" - ), - "", - ] - lines = _format_dbt_section(lines=lines, plan=plan, options=options) - lines.append("") - lines = _format_sqlbuild_section(lines=lines, plan=plan, options=options, use_color=use_color) - lines = _format_anchor_section(lines=lines, plan=plan, options=options) - lines = _format_path_translation_section(lines=lines, plan=plan) - lines = _format_warning_section(lines=lines, plan=plan) - result: str = "\n".join(lines) - return result if use_color else _ANSI_ESCAPE_PATTERN.sub("", result) - - -def format_dbt_interop_plan_json(plan: DbtInteropPlan) -> str: - """Serialize an ordinary interop selection plan to stable JSON.""" - - result: dict[str, object] = { - "command": plan.command.value, - "dbt": { - "argv": list(plan.dbt_command_argv), - "supplemental_argvs": [list(argv) for argv in plan.supplemental_dbt_command_argvs], - "selected_unique_ids": list(plan.dbt_selected_unique_ids), - "required_unique_ids": list(plan.selection.dbt_required_unique_ids), - "required_selector_terms": list(plan.dbt_required_selector_terms), - "skipped": plan.dbt_skip_reason is not None, - "skip_reason": plan.dbt_skip_reason.value if plan.dbt_skip_reason is not None else None, - }, - "sqlbuild": { - "argvs": [list(argv) for argv in plan.sqlbuild_command_argvs], - "selected_models": list(plan.selection.sqlbuild_model_names), - "skipped": plan.sqlbuild_skip_reason is not None, - "skip_reason": ( - plan.sqlbuild_skip_reason.value if plan.sqlbuild_skip_reason is not None else None - ), - }, - "anchors": [ - { - "term": term, - "unique_ids": list(plan.selection.dbt_anchor_unique_ids_by_term.get(term, ())), - } - for term in plan.selection.dbt_anchor_terms - ], - "path_translations": [ - {"from": original, "to": translated} - for original, translated in plan.selection.path_translations - ], - "warnings": list(plan.warnings), - } - return json.dumps(result, indent=2) - - -def _format_dbt_section( - *, lines: list[str], plan: DbtInteropPlan, options: DisplayOptions -) -> list[str]: - style: CliStyle = CliStyle(use_color=True) - selected_count: int = len(plan.dbt_selected_unique_ids) - required_count: int = len(plan.selection.dbt_required_unique_ids) - lines.append( - f"{style.dbt_label('dbt')} " - f"{style.dbt_section(f'({selected_count} selected, {required_count} required)')}" - ) - if plan.dbt_skip_reason is not None: - lines.append(style.muted(" skipped: no dbt work selected")) - return lines - lines.append(f" {style.label('command')}: {style.command(' '.join(plan.dbt_command_argv))}") - nodes_by_type: dict[str, list[DbtLsNode]] = defaultdict(list) - for node in sorted(plan.dbt_selected_nodes, key=_dbt_node_sort_key): - nodes_by_type[node.resource_type or "resource"].append(node) - for resource_type, nodes in sorted(nodes_by_type.items()): - lines.append("") - lines.append(f" {style.plan_section(f'{resource_type} ({len(nodes)})')}") - visible_nodes: Sequence[DbtLsNode] = visible_entries(entries=nodes, options=options) - node_index: int - for node_index, node in enumerate(visible_nodes): - connector: str = tree_connector( - style=style, - last=node_index == len(visible_nodes) - 1 and len(nodes) <= len(visible_nodes), - ) - lines.append(f" {connector} {style.dbt_object_name(_dbt_node_display_name(node))}") - lines = append_overflow_line( - lines=lines, - total_count=len(nodes), - visible_count=len(visible_nodes), - indent=" ", - options=options, - ) - if required_count: - lines.append("") - lines.append(style.dbt_label(f" required ({required_count})")) - required_ids: Sequence[str] = visible_entries( - entries=plan.selection.dbt_required_unique_ids, - options=options, - ) - unique_id_index: int - for unique_id_index, unique_id in enumerate(required_ids): - connector = tree_connector(style=style, last=unique_id_index == len(required_ids) - 1) - lines.append(f" {connector} {style.dbt_object_name(unique_id)}") - return lines - - -def _format_sqlbuild_section( - *, - lines: list[str], - plan: DbtInteropPlan, - options: DisplayOptions, - use_color: bool, -) -> list[str]: - style: CliStyle = CliStyle(use_color=True) - lines.append(style.plan_section(f"SQLBuild ({len(plan.selection.sqlbuild_model_names)})")) - if plan.sqlbuild_skip_reason is not None: - lines.append(style.muted(" skipped: no SQLBuild work selected")) - return lines - if plan.sqlbuild_plan_output is not None: - rendered: str = format_plan( - plan=plan.sqlbuild_plan_output, - use_color=use_color, - display_options=options, - ) - lines.extend(f" {line}" if line else "" for line in rendered.splitlines()) - return lines - for name in visible_entries(entries=plan.selection.sqlbuild_model_names, options=options): - lines.append(f" {style.object_name(name)}") - return lines - - -def _format_anchor_section( - *, lines: list[str], plan: DbtInteropPlan, options: DisplayOptions -) -> list[str]: - if not plan.selection.dbt_anchor_terms: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.extend(("", style.label("dbt selector anchors"))) - visible_terms: Sequence[str] = visible_entries( - entries=plan.selection.dbt_anchor_terms, - options=options, - ) - for term in visible_terms: - count: int = len(plan.selection.dbt_anchor_unique_ids_by_term.get(term, ())) - lines.append(f" {style.command(term)}: {count} dbt resources") - return lines - - -def _format_path_translation_section(*, lines: list[str], plan: DbtInteropPlan) -> list[str]: - if not plan.selection.path_translations: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.extend(("", style.label("path translations"))) - for original, translated in plan.selection.path_translations: - lines.append(f" {style.command(original)} -> {style.command(translated)}") - return lines - - -def _format_warning_section(*, lines: list[str], plan: DbtInteropPlan) -> list[str]: - if not plan.warnings: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.extend(("", style.warning("Warnings"))) - for warning in plan.warnings: - lines.append(style.warning(f" {warning}")) - return lines - - -def _dbt_node_display_name(node: DbtLsNode) -> str: - if node.fqn: - return ".".join(node.fqn) - return node.name or node.unique_id - - -def _dbt_node_sort_key(node: DbtLsNode) -> tuple[str, str]: - return node.resource_type or "", _dbt_node_display_name(node) diff --git a/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py b/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py index f2d81ace86..6f47f24aa8 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py +++ b/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py @@ -5,6 +5,7 @@ import sys from pathlib import Path +from sqlbuild._native import normalize_dbt_profile_connection from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.profile.load import ( @@ -13,7 +14,6 @@ load_raw_dbt_profile, select_dbt_profile_output, ) -from sqlbuild.integrations.dbt._helpers.profile.normalize import normalize_dbt_profile_connection from sqlbuild.integrations.dbt._helpers.profile.render import render_selected_dbt_profile_output from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import ( diff --git a/src/sqlbuild/integrations/dbt/_helpers/profile/init.py b/src/sqlbuild/integrations/dbt/_helpers/profile/init.py index de6fd49957..7eb32dcdf6 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/profile/init.py +++ b/src/sqlbuild/integrations/dbt/_helpers/profile/init.py @@ -6,13 +6,13 @@ import subprocess from pathlib import Path +from sqlbuild._native import build_dbt_init_project_toml, normalize_dbt_profile_connection from sqlbuild.integrations.dbt._helpers.profile.load import ( default_profiles_dir, load_dbt_project_metadata, load_raw_dbt_profile, select_dbt_profile_output, ) -from sqlbuild.integrations.dbt._helpers.profile.normalize import normalize_dbt_profile_connection from sqlbuild.integrations.dbt._helpers.profile.render import render_selected_dbt_profile_output from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import ( @@ -69,7 +69,7 @@ def build_dbt_init_project(*, request: DbtInitRequest) -> DbtInitResult: ) normalized: NormalizedDbtProfileConnection = normalize_dbt_profile_connection(resolved=resolved) target_path: Path = _resolve_path(root=dbt_project_dir, path=Path(metadata.target_path)) - toml: str = _build_project_toml( + toml: str = build_dbt_init_project_toml( project_name=metadata.project_name.replace("-", "_"), adapter=normalized.adapter, target_name=selected.target_name, @@ -178,54 +178,6 @@ def _display_path(*, path: Path, root: Path) -> str: return path.as_posix() -def _build_project_toml( - *, - project_name: str, - adapter: str, - target_name: str, - profile_name: str, - dbt_project_dir: str, - profiles_dir: str, - target_path: str, - target_schema: str | None, - target_database: str | None, -) -> str: - connection_name: str = f"{target_name}_dbt_profile" - target_lines: list[str] = [ - f"[connections.{_quote_key(connection_name)}]", - 'source = "dbt_profile"', - f'profile = "{_escape(profile_name)}"', - f'target = "{_escape(target_name)}"', - "", - f"[targets.{_quote_key(target_name)}]", - f'connection = "{_escape(connection_name)}"', - ] - if target_database is not None: - target_lines.append(f'database = "{_escape(target_database)}"') - if target_schema is not None: - target_lines.append(f'schema = "{_escape(target_schema)}"') - return ( - f'name = "{_escape(project_name)}"\n' - f'adapter = "{_escape(adapter)}"\n' - f'default_target = "{_escape(target_name)}"\n\n' - "[dbt]\n" - f'project_dir = "{_escape(dbt_project_dir)}"\n' - f'profiles_dir = "{_escape(profiles_dir)}"\n' - f'target_path = "{_escape(target_path)}"\n' - f'target = "{_escape(target_name)}"\n\n' + "\n".join(target_lines) + "\n" - ) - - -def _quote_key(value: str) -> str: - if value.replace("_", "").replace("-", "").isalnum(): - return value - return f'"{_escape(value)}"' - - -def _escape(value: str) -> str: - return value.replace("\\", "\\\\").replace('"', '\\"') - - def _dbt_debug_warnings( *, dbt_project_dir: Path, profiles_dir: Path, target_name: str ) -> tuple[str, ...]: diff --git a/src/sqlbuild/integrations/dbt/_helpers/profile/normalize.py b/src/sqlbuild/integrations/dbt/_helpers/profile/normalize.py deleted file mode 100644 index 24176361c4..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/profile/normalize.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Normalize rendered dbt profile outputs into SQLBuild connection config.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.models import ( - NormalizedDbtProfileConnection, - ResolvedDbtProfileOutput, -) - -_DBT_BIGQUERY_SUPPORTED_METHODS: frozenset[str] = frozenset({"oauth", "service-account"}) -_DBT_BIGQUERY_UNSUPPORTED_METHODS: frozenset[str] = frozenset( - {"oauth-secrets", "service-account-json", "external-oauth-wif"} -) -_DBT_DATABRICKS_PAT_AUTH_TYPE: str = "pat" -_DBT_DUCKDB_UNSUPPORTED_PROFILE_KEYS: frozenset[str] = frozenset( - { - "config_options", - "disable_transactions", - "external_root", - "filesystems", - "is_ducklake", - "keep_open", - "module_paths", - "plugins", - "remote", - "retries", - "secrets", - "use_credential_provider", - } -) -_DBT_PROFILE_HOST_KEY: str = "host" -_DBT_PROFILE_HTTP_PATH_KEY: str = "http_path" -_DBT_PROFILE_KEYFILE_KEY: str = "keyfile" -_DBT_PROFILE_LOCATION_KEY: str = "location" -_DBT_PROFILE_SCHEMA_KEY: str = "schema" -_DBT_PROFILE_TOKEN_KEY: str = "token" -_DBT_SQLSERVER_SQL_AUTHENTICATION: str = "sql" - - -def normalize_dbt_profile_connection( - *, resolved: ResolvedDbtProfileOutput -) -> NormalizedDbtProfileConnection: - """Normalize rendered dbt profile output for SQLBuild.""" - - adapter_type: str = resolved.adapter_type - try: - builtin_adapter: BuiltinAdapter = BuiltinAdapter(adapter_type) - except ValueError as exc: - raise DbtProfileError( - f"dbt profile type '{adapter_type or ''}' is not supported by SQLBuild " - "dbt-profile connection resolution yet. Currently supported: " - "bigquery, databricks, duckdb, postgres, snowflake, sqlserver" - ) from exc - match builtin_adapter: - case BuiltinAdapter.DUCKDB: - return _normalize_duckdb(resolved=resolved) - case BuiltinAdapter.POSTGRES: - return _normalize_postgres(resolved=resolved) - case BuiltinAdapter.SNOWFLAKE: - return _normalize_snowflake(resolved=resolved) - case BuiltinAdapter.BIGQUERY: - return _normalize_bigquery(resolved=resolved) - case BuiltinAdapter.DATABRICKS: - return _normalize_databricks(resolved=resolved) - case BuiltinAdapter.SQLSERVER: - return _normalize_sqlserver(resolved=resolved) - case BuiltinAdapter.MOTHERDUCK: - raise DbtProfileError( - "dbt profile type 'motherduck' is not supported directly. Use a dbt-duckdb " - "profile with a MotherDuck path after SQLBuild MotherDuck profile support lands." - ) - - -def _normalize_duckdb(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - connection: dict[str, object] = {} - path_value: object | None = output.get("path") - database_value: object | None = output.get("database") - database: object = path_value if path_value is not None else database_value - if database is None: - database = ":memory:" - connection["database"] = database - for key in ("extensions", "settings", "attach"): - if key in output: - connection[key] = output[key] - warnings: list[str] = [] - unsupported_keys: tuple[str, ...] = tuple( - sorted(key for key in output if key in _DBT_DUCKDB_UNSUPPORTED_PROFILE_KEYS) - ) - if unsupported_keys: - warnings.append( - "dbt-duckdb profile fields are not applied by SQLBuild yet: " - + ", ".join(unsupported_keys) - ) - schema_value: object | None = output.get("schema") - target_schema: str | None = schema_value if isinstance(schema_value, str) else None - database_name: str | None = database_value if isinstance(database_value, str) else None - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.DUCKDB.value, - connection=connection, - target_schema=target_schema, - target_database=database_name, - warnings=tuple(warnings), - ) - - -def _normalize_postgres(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - connection: dict[str, object] = _copy_present( - output=output, - keys=( - "host", - "port", - "user", - "connect_timeout", - "role", - "keepalives_idle", - "sslmode", - "sslcert", - "sslkey", - "sslrootcert", - "application_name", - ), - ) - database: object | None = output.get("dbname", output.get("database")) - password: object | None = output.get("pass", output.get("password")) - if database is not None: - connection["dbname"] = database - if password is not None: - connection["password"] = password - target_schema: str | None = _string_or_none(output.get("schema")) - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.POSTGRES.value, - connection=connection, - target_schema=target_schema, - target_database=None, - ) - - -def _normalize_snowflake(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - connection: dict[str, object] = dict(output) - connection.pop("type", None) - connection.pop("threads", None) - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.SNOWFLAKE.value, - connection=connection, - target_schema=_string_or_none(output.get("schema")), - target_database=_string_or_none(output.get("database")), - ) - - -def _normalize_bigquery(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - method: str | None = _string_or_none(output.get("method")) - if method in _DBT_BIGQUERY_UNSUPPORTED_METHODS: - raise DbtProfileError( - "dbt BigQuery profile method " - f"'{method}' is not supported by SQLBuild dbt-profile normalization yet. " - "Supported methods: oauth, service-account" - ) - project: object | None = output.get("project", output.get("database")) - connection: dict[str, object] = {} - if project is not None: - connection["project"] = project - if _DBT_PROFILE_LOCATION_KEY in output: - connection["location"] = output[_DBT_PROFILE_LOCATION_KEY] - if _DBT_PROFILE_KEYFILE_KEY in output: - connection["credentials_path"] = output[_DBT_PROFILE_KEYFILE_KEY] - if method is not None and method not in _DBT_BIGQUERY_SUPPORTED_METHODS: - raise DbtProfileError( - "dbt BigQuery profile method " - f"'{method}' is not supported by SQLBuild dbt-profile normalization yet. " - "Supported methods: oauth, service-account" - ) - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.BIGQUERY.value, - connection=connection, - target_schema=_string_or_none(output.get("dataset", output.get("schema"))), - target_database=_string_or_none(project), - ) - - -def _normalize_databricks(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - unsupported_auth_keys: tuple[str, ...] = tuple( - sorted( - key - for key in ( - "client_id", - "client_secret", - "azure_client_id", - "azure_client_secret", - "oauth_redirect_url", - "oauth_scopes", - ) - if output.get(key) is not None - ) - ) - auth_type: str | None = _string_or_none(output.get("auth_type")) - if auth_type is not None and auth_type != _DBT_DATABRICKS_PAT_AUTH_TYPE: - raise DbtProfileError( - "dbt Databricks profile auth_type " - f"'{auth_type}' is not supported by SQLBuild dbt-profile normalization yet. " - "Supported auth: personal access token" - ) - if unsupported_auth_keys: - raise DbtProfileError( - "dbt Databricks OAuth/Azure profile fields are not supported by SQLBuild " - "dbt-profile normalization yet: " + ", ".join(unsupported_auth_keys) - ) - connection: dict[str, object] = {} - if _DBT_PROFILE_HOST_KEY in output: - connection["server_hostname"] = output[_DBT_PROFILE_HOST_KEY] - if _DBT_PROFILE_HTTP_PATH_KEY in output: - connection["http_path"] = output[_DBT_PROFILE_HTTP_PATH_KEY] - if _DBT_PROFILE_TOKEN_KEY in output: - connection["token"] = output[_DBT_PROFILE_TOKEN_KEY] - catalog: object | None = output.get("catalog", output.get("database")) - if catalog is None: - catalog = "hive_metastore" - connection["catalog"] = catalog - if _DBT_PROFILE_SCHEMA_KEY in output: - connection["schema"] = output[_DBT_PROFILE_SCHEMA_KEY] - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.DATABRICKS.value, - connection=connection, - target_schema=_string_or_none(output.get("schema")), - target_database=_string_or_none(catalog), - ) - - -def _normalize_sqlserver(*, resolved: ResolvedDbtProfileOutput) -> NormalizedDbtProfileConnection: - output: dict[str, object] = dict(resolved.output) - if output.get("windows_login") is True or output.get("trusted_connection") is True: - raise DbtProfileError( - "dbt SQL Server Windows authentication is not supported by SQLBuild " - "dbt-profile normalization yet" - ) - authentication: str | None = _string_or_none(output.get("authentication", output.get("auth"))) - if authentication is not None and authentication.lower() != _DBT_SQLSERVER_SQL_AUTHENTICATION: - raise DbtProfileError( - "dbt SQL Server authentication mode " - f"'{authentication}' is not supported by SQLBuild dbt-profile normalization yet. " - "Supported authentication: sql" - ) - connection: dict[str, object] = {} - host: object | None = output.get("server", output.get("host")) - user: object | None = output.get("UID", output.get("user", output.get("username"))) - password: object | None = output.get("PWD", output.get("password", output.get("pass"))) - for key, value in ( - ("host", host), - ("port", output.get("port")), - ("user", user), - ("password", password), - ("database", output.get("database")), - ): - if value is not None: - connection[key] = value - return NormalizedDbtProfileConnection( - adapter=BuiltinAdapter.SQLSERVER.value, - connection=connection, - target_schema=_string_or_none(output.get("schema")), - target_database=None, - ) - - -def _copy_present(*, output: dict[str, object], keys: tuple[str, ...]) -> dict[str, object]: - return {key: output[key] for key in keys if key in output} - - -def _string_or_none(value: object | None) -> str | None: - return value if isinstance(value, str) and value else None diff --git a/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py b/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py index 9043efdb0f..fd2892daf4 100644 --- a/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py +++ b/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py @@ -2,17 +2,13 @@ from pathlib import Path -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.integrations.dbt.main.manifest._resolve_reference_relation import ( +from sqlbuild._native import ( resolve_dbt_reference_relation, + validate_compile_dbt_model_names, + validate_compile_dbt_model_reference, ) -from sqlbuild.integrations.dbt.main.manifest._validate_compile_model_names import ( - validate_compile_model_names, -) -from sqlbuild.integrations.dbt.main.manifest._validate_compile_model_reference import ( - validate_compile_model_reference, -) +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.integrations.dbt.models import DbtManifestIndex @@ -23,7 +19,7 @@ def __init__(self, *, dbt_manifest: DbtManifestIndex | None) -> None: self._dbt_manifest = dbt_manifest def validate_model_names(self, *, known_model_names: set[str]) -> None: - validate_compile_model_names( + validate_compile_dbt_model_names( known_model_names=known_model_names, dbt_manifest=self._dbt_manifest, ) @@ -102,7 +98,7 @@ def validate_reference( ref_package: str | None, owner_relative_sql_path: Path, ) -> None: - validate_compile_model_reference( + validate_compile_dbt_model_reference( reference=CompileSqlReference( ref_kind=ref_kind, ref_name=ref_name, diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_build_compile_manifest_index.py b/src/sqlbuild/integrations/dbt/main/manifest/_build_compile_manifest_index.py deleted file mode 100644 index 2779ca44f4..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_build_compile_manifest_index.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public dbt compile manifest index entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.manifest.compile_refs import ( - build_compile_dbt_manifest_index, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def build_compile_manifest_index(*, manifest_contents: str | None) -> DbtManifestIndex | None: - """Build a dbt manifest index from discovered manifest contents.""" - - return build_compile_dbt_manifest_index(manifest_contents=manifest_contents) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_load_manifest_index.py b/src/sqlbuild/integrations/dbt/main/manifest/_load_manifest_index.py deleted file mode 100644 index df2f1281cf..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_load_manifest_index.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public dbt manifest entrypoints.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.integrations.dbt._helpers.manifest.core import ( - load_dbt_manifest_index, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def load_manifest_index(*, manifest_path: Path) -> DbtManifestIndex: - """Load and index a dbt manifest file.""" - - return load_dbt_manifest_index(manifest_path=manifest_path) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_manifest_model_materialization.py b/src/sqlbuild/integrations/dbt/main/manifest/_manifest_model_materialization.py deleted file mode 100644 index 9d744d6292..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_manifest_model_materialization.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Resolve dbt manifest model materialization.""" - -from sqlbuild.integrations.dbt._helpers.manifest.core import ( - dbt_manifest_model_materialization as _resolve, -) -from sqlbuild.integrations.dbt.models import DbtManifestModel - - -def dbt_manifest_model_materialization(*, model: DbtManifestModel) -> str | None: - """Return a manifest model's normalized materialization.""" - - return _resolve(model=model) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_reference_relation.py b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_reference_relation.py deleted file mode 100644 index cc96627b27..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_reference_relation.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Resolve a dbt reference relation.""" - -from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( - resolve_dbt_reference_relation as _resolve, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def resolve_dbt_reference_relation( - *, - manifest: DbtManifestIndex | None, - ref_kind: str, - ref_name: str, - ref_package: str | None, -) -> str | None: - """Resolve one external dbt reference to its relation.""" - - return _resolve( - manifest=manifest, - ref_kind=ref_kind, - ref_name=ref_name, - ref_package=ref_package, - ) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py deleted file mode 100644 index 22d3daf843..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Resolve SQLBuild model references to dbt models.""" - -from collections.abc import Sequence - -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( - resolve_sqlbuild_model_dbt_refs as _resolve, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex, DbtManifestModel - - -def resolve_sqlbuild_model_dbt_refs( - *, - project: CompiledProjectView, - manifest: DbtManifestIndex, - selected_model_names: Sequence[str] | None = None, -) -> tuple[tuple[CompiledModelView, DbtManifestModel], ...]: - """Return dbt refs used directly by SQLBuild models.""" - - return _resolve(project=project, manifest=manifest, selected_model_names=selected_model_names) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_names.py b/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_names.py deleted file mode 100644 index 2a7386338c..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_names.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public dbt compile duplicate model-name validation entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.manifest.compile_refs import ( - validate_compile_dbt_model_names, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def validate_compile_model_names( - *, known_model_names: set[str], dbt_manifest: DbtManifestIndex | None -) -> None: - """Reject dbt/SQLBuild duplicate logical model names.""" - - validate_compile_dbt_model_names( - known_model_names=known_model_names, - dbt_manifest=dbt_manifest, - ) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_reference.py b/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_reference.py deleted file mode 100644 index e17fc11e99..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_validate_compile_model_reference.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public dbt compile model ref validation entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.integrations.dbt._helpers.manifest.compile_refs import ( - validate_compile_dbt_model_reference, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def validate_compile_model_reference( - *, - reference: CompileSqlReference, - model_relative_path: Path, - dbt_manifest: DbtManifestIndex | None, -) -> None: - """Validate a SQLBuild model __dbt_ref() against a dbt manifest.""" - - validate_compile_dbt_model_reference( - reference=reference, - model_relative_path=model_relative_path, - dbt_manifest=dbt_manifest, - ) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py b/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py index cdac9b4eb6..bb39ded142 100644 --- a/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py +++ b/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py @@ -2,13 +2,11 @@ from __future__ import annotations +from sqlbuild._native import build_compile_dbt_manifest_index from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver import ( DbtCompileReferenceResolver, ) -from sqlbuild.integrations.dbt.main.manifest._build_compile_manifest_index import ( - build_compile_manifest_index, -) def build_compile_reference_resolver( @@ -19,5 +17,5 @@ def build_compile_reference_resolver( if manifest_contents is None: return None return DbtCompileReferenceResolver( - dbt_manifest=build_compile_manifest_index(manifest_contents=manifest_contents) + dbt_manifest=build_compile_dbt_manifest_index(manifest_contents=manifest_contents) ) diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/render_plan.py b/src/sqlbuild/integrations/dbt/main/pipeline/render_plan.py deleted file mode 100644 index 2023e70b38..0000000000 --- a/src/sqlbuild/integrations/dbt/main/pipeline/render_plan.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Render dbt interop plan output for CLI callers.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt.main.planning._render_interop_plan_json import ( - format_dbt_interop_plan_json, -) -from sqlbuild.integrations.dbt.main.planning._render_interop_plan_text import ( - format_dbt_interop_plan, -) -from sqlbuild.integrations.dbt.models import DbtInteropPlan -from sqlbuild.presentation.models import DisplayOptions - - -def render_dbt_interop_plan( - *, - plan: DbtInteropPlan, - json_output: bool, - use_color: bool, - display_options: DisplayOptions | None = None, -) -> str: - """Render a dbt interop plan in the requested CLI output format.""" - - if json_output: - return format_dbt_interop_plan_json(plan) - return "\n" + format_dbt_interop_plan( - plan=plan, use_color=use_color, display_options=display_options - ) diff --git a/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_json.py b/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_json.py deleted file mode 100644 index 10768e319a..0000000000 --- a/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_json.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Format a dbt interop plan as JSON.""" - -from sqlbuild.integrations.dbt._helpers.planning.plan import format_dbt_interop_plan_json as _format -from sqlbuild.integrations.dbt.models import DbtInteropPlan - - -def format_dbt_interop_plan_json(plan: DbtInteropPlan) -> str: - """Serialize a dbt interop plan to stable JSON.""" - - return _format(plan) diff --git a/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_text.py b/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_text.py deleted file mode 100644 index 99cb325cc9..0000000000 --- a/src/sqlbuild/integrations/dbt/main/planning/_render_interop_plan_text.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Format a dbt interop plan.""" - -from sqlbuild.integrations.dbt._helpers.planning.plan import format_dbt_interop_plan as _format -from sqlbuild.integrations.dbt.models import DbtInteropPlan -from sqlbuild.presentation.models import DisplayOptions - - -def format_dbt_interop_plan( - *, - plan: DbtInteropPlan, - use_color: bool = True, - display_options: DisplayOptions | None = None, -) -> str: - """Format a dbt interop plan for human output.""" - - return _format(plan=plan, use_color=use_color, display_options=display_options) diff --git a/src/sqlbuild/presentation/_helpers/structure.py b/src/sqlbuild/presentation/_helpers/structure.py index 89f34f3ef6..dfe6e82998 100644 --- a/src/sqlbuild/presentation/_helpers/structure.py +++ b/src/sqlbuild/presentation/_helpers/structure.py @@ -4,8 +4,6 @@ from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.constants import ( - PHASE_FAIL_GLYPH, - PHASE_OK_GLYPH, TREE_BRANCH_GLYPH, TREE_LAST_GLYPH, ) @@ -24,14 +22,3 @@ def format_surface_header(*, style: CliStyle, title: str, context: str | None = if context: rendered = f"{rendered} {style.muted(context)}" return rendered - - -def format_phase_line(*, style: CliStyle, ok: bool, label: str, summary: str | None = None) -> str: - """Render a concise phase-completion line: state glyph, label, dim summary.""" - - glyph: str = style.success(PHASE_OK_GLYPH) if ok else style.error(PHASE_FAIL_GLYPH) - rendered_label: str = label if ok else style.error_strong(label) - rendered: str = f"{glyph} {rendered_label}" - if summary: - rendered = f"{rendered} {style.muted(summary)}" - return rendered diff --git a/src/sqlbuild/presentation/main/phase_line.py b/src/sqlbuild/presentation/main/phase_line.py deleted file mode 100644 index 2c6b3796bc..0000000000 --- a/src/sqlbuild/presentation/main/phase_line.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public phase-completion line rendering entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.structure import format_phase_line as _format_phase_line -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_phase_line(*, style: CliStyle, ok: bool, label: str, summary: str | None = None) -> str: - """Render a concise phase-completion line: state glyph, label, dim summary.""" - - return _format_phase_line(style=style, ok=ok, label=label, summary=summary) diff --git a/src/sqlbuild/presentation/main/surface_header.py b/src/sqlbuild/presentation/main/surface_header.py deleted file mode 100644 index 2764efdca4..0000000000 --- a/src/sqlbuild/presentation/main/surface_header.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public command surface header rendering entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.structure import format_surface_header as _format_surface_header -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_surface_header(*, style: CliStyle, title: str, context: str | None = None) -> str: - """Render a command surface header: accent title plus dim context.""" - - return _format_surface_header(style=style, title=title, context=context) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py index 1ece17e899..852339009f 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py @@ -9,18 +9,14 @@ from importlib.machinery import ModuleSpec from pathlib import Path from types import ModuleType -from typing import cast +from sqlbuild._native import builtin_rules from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine._helpers.engine.native import native_catalogue, native_selected_codes +from sqlbuild.rule_engine._helpers.engine.native import native_selected_codes from sqlbuild.rule_engine.constants import ( - CONTRACT_BOOLEAN_NAME_CODE, - CONTRACT_DATE_NAME_CODE, - CONTRACT_TIMESTAMP_NAME_CODE, - EMPTY_INPUT_TEST_CODE, INIT_MODULE_NAME, RULE_DECORATOR_TOKEN, ) @@ -28,11 +24,9 @@ from sqlbuild.rule_engine.models import ( Finding, Rule, - RuleGuidance, RuleOption, RulesConfig, ) -from sqlbuild.rule_engine.types import RuleSubject def build_catalogue( @@ -65,107 +59,7 @@ def build_catalogue( def _builtins() -> tuple[Rule, ...]: - return tuple( - Rule( - code=str(item["code"]), - family=str(item["code"])[:-3], - slug=str(item["slug"]), - message=str(item["message"]), - remediation=str(item["remediation"]), - check=_native_builtin, - options=_builtin_options(str(item["code"])), - enabled_by_default=bool(item["enabled_by_default"]), - subject=(RuleSubject.PROJECT if bool(item["project_wide"]) else RuleSubject.MODEL), - guidance=_guidance(item.get("guidance")), - ) - for item in native_catalogue() - ) - - -def _builtin_options(code: str) -> tuple[RuleOption[object], ...]: - if code == CONTRACT_BOOLEAN_NAME_CODE: - return ( - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_numeric_indicators", - default=False, - description="Accept numeric is_/has_/can_ indicator columns.", - ), - ), - ) - if code == CONTRACT_TIMESTAMP_NAME_CODE: - return ( - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_date_for_at", - default=False, - description="Accept DATE for columns ending in _at.", - ), - ), - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_numeric_epoch", - default=False, - description="Accept numeric epoch values for timestamp-named columns.", - ), - ), - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_encoded_values", - default=False, - description="Accept string or variant encoded timestamp values.", - ), - ), - ) - if code == CONTRACT_DATE_NAME_CODE: - return ( - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_timestamps", - default=False, - description="Accept timestamp types for columns ending in _date.", - ), - ), - cast( - RuleOption[object], - RuleOption.boolean( - name="allow_encoded_values", - default=False, - description="Accept string or variant encoded date values.", - ), - ), - ) - if code == EMPTY_INPUT_TEST_CODE: - return ( - cast( - RuleOption[object], - RuleOption.string_list( - name="allowed_tests", - default=(), - description=( - "Reviewed SQL test names that may assert only that empty inputs produce " - "no rows; listed tests count toward min_tests_per_model." - ), - ), - ), - ) - return () - - -def _guidance(value: object) -> RuleGuidance | None: - if not isinstance(value, dict): - return None - payload: dict[str, object] = {str(key): item for key, item in value.items()} - return RuleGuidance( - good_example=str(payload["good_example"]), - anti_tautology=str(payload["anti_tautology"]), - mutation_check=str(payload["mutation_check"]), - ) + return tuple(builtin_rules(check=_native_builtin)) def _native_builtin(*, model: object, ctx: object) -> list[Finding]: diff --git a/src/sqlbuild/rule_engine/_helpers/engine/config.py b/src/sqlbuild/rule_engine/_helpers/engine/config.py index 383f54139b..17c5eea811 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/config.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/config.py @@ -1,139 +1,16 @@ -"""Native-owned strict rules configuration loading.""" +"""Resolve rule ignore selectors against the compiled project.""" from __future__ import annotations from dataclasses import replace from pathlib import Path -from typing import cast -from sqlbuild._native import format_count_noun from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME from sqlbuild.compiler.planner.main.selection.scope import build_planner_scope from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.errors.setting_help.main.join_helps import join_helps -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.errors.setting_help.main.setting_note import setting_note -from sqlbuild.rule_engine._helpers.engine.native import load_native_config -from sqlbuild.rule_engine.constants import ( - ALLOW_EXCEPTIONS_KEY, - RULE_EXCEPTIONS_KEY, - RULE_IGNORES_KEY, - RULES_CONFIG_SECTION, -) -from sqlbuild.rule_engine.exceptions import RulesConfigError, RulesError -from sqlbuild.rule_engine.models import ( - GraphEdgeExclusion, - LayoutConfig, - RuleExemption, - RuleIgnore, - RulesCacheConfig, - RulesConfig, - SelectStarAllow, - SqlTestRulesConfig, - ThresholdOverride, -) -from sqlbuild.rule_engine.types import RuleOptionValue - - -def load_rules_config(project_dir: Path) -> RulesConfig: - """Load configuration validated and normalized by the native engine.""" - - payload: dict[str, object] = load_native_config(project_dir) - config: RulesConfig = RulesConfig( - select=_strings(payload.get("select")), - ignore=_strings(payload.get("ignore")), - thresholds=_integers(payload.get("thresholds")), - threshold_overrides=tuple( - ThresholdOverride( - paths=_strings(item.get("paths")), - thresholds=_integers(item.get("thresholds")), - reason=str(item["reason"]), - ) - for item in _tables(payload.get("threshold_overrides")) - ), - rule_options=_rule_options(payload.get("rule_options")), - rule_exceptions=tuple( - RuleExemption( - rule=str(item["rule"]), path=str(item["path"]), reason=str(item["reason"]) - ) - for item in _tables(payload.get("rule_exceptions")) - ), - graph_edge_exceptions=tuple( - GraphEdgeExclusion( - consumer=str(item["consumer"]), - dependency=str(item["dependency"]), - reason=str(item["reason"]), - ) - for item in _tables(payload.get("graph_edge_exceptions")) - ), - rule_ignores=tuple( - RuleIgnore( - rules=_strings(item.get("rules")), - paths=_strings(item.get("paths")), - reason=str(item["reason"]), - selectors=_strings(item.get("selectors")), - ) - for item in _tables(payload.get("rule_ignores")) - ), - allow_exceptions=payload.get("allow_exceptions") is not False, - select_star_allow=tuple( - SelectStarAllow(paths=_strings(item.get("paths")), reason=str(item["reason"])) - for item in _tables(payload.get("select_star_allow")) - ), - domains=_strings(payload.get("domains")), - approved_source_tokens=_strings(payload.get("approved_source_tokens")), - retired_source_tokens=_string_mapping(payload.get("retired_source_tokens")), - sql_tests=_sql_tests(payload.get("sql_tests")), - layout=_layout(payload.get("layout")), - cache=_cache(payload.get("cache")), - ) - _reject_forbidden_exceptions(config) - return config - - -def _reject_forbidden_exceptions(config: RulesConfig) -> None: - """Fail when `[rules] allow_exceptions = false` and exceptions or scoped ignores exist.""" - - if config.allow_exceptions: - return - entries: list[str] = [] - tables: list[str] = [] - for key, count in ( - (RULE_EXCEPTIONS_KEY, len(config.rule_exceptions)), - (RULE_IGNORES_KEY, len(config.rule_ignores)), - ): - if count: - table: str = f"[[{RULES_CONFIG_SECTION}.{key}]]" - tables.append(table) - entries.append( - format_count_noun(count=count, singular=f"{table} entry", plural=f"{table} entries") - ) - if not entries: - return - raise RulesConfigError( - f"{PROJECT_CONFIG_FILENAME} has {' and '.join(entries)}, which this project does not " - "allow; " - + setting_note( - file_name=PROJECT_CONFIG_FILENAME, - section=RULES_CONFIG_SECTION, - key=ALLOW_EXCEPTIONS_KEY, - value=False, - ), - help=join_helps( - f"remove every {' and '.join(tables)} entry and fix the findings it hides; " - f"to turn a Rule off for the whole project, add its code to " - f"[{RULES_CONFIG_SECTION}] ignore", - setting_help( - purpose="to allow Rule exceptions, scoped ignores and inline suppressions", - file_name=PROJECT_CONFIG_FILENAME, - section=RULES_CONFIG_SECTION, - key=ALLOW_EXCEPTIONS_KEY, - value=True, - ), - ), - ) +from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild.rule_engine.models import RuleIgnore, RulesConfig def resolve_rule_ignore_selectors( @@ -173,71 +50,3 @@ def _resource_paths(*, project: CompiledProjectView) -> dict[CompiledObjectKey, paths.update({seed.key: seed.seed_file.relative_path for seed in project.seeds}) paths.update({source.key: source.source_file.relative_path for source in project.sources}) return paths - - -def _strings(value: object) -> tuple[str, ...]: - return tuple(str(item) for item in value) if isinstance(value, list) else () - - -def _tables(value: object) -> tuple[dict[str, object], ...]: - if not isinstance(value, list): - return () - tables: list[dict[str, object]] = [] - for table in value: - if isinstance(table, dict): - tables.append({str(key): item for key, item in table.items()}) - return tuple(tables) - - -def _integers(value: object) -> dict[str, int]: - if not isinstance(value, dict): - return {} - return {str(key): item for key, item in value.items() if isinstance(item, int)} - - -def _string_mapping(value: object) -> dict[str, str]: - if not isinstance(value, dict): - return {} - return {str(key): str(item) for key, item in value.items()} - - -def _rule_options(value: object) -> dict[str, dict[str, RuleOptionValue]]: - if not isinstance(value, dict): - return {} - result: dict[str, dict[str, RuleOptionValue]] = {} - for code, raw_options in value.items(): - if not isinstance(raw_options, dict): - continue - options: dict[str, RuleOptionValue] = {} - for name, item in raw_options.items(): - if isinstance(item, list): - options[str(name)] = cast("RuleOptionValue", tuple(item)) - elif isinstance(item, (bool, int, str)): - options[str(name)] = item - result[str(code)] = options - return result - - -def _cache(value: object) -> RulesCacheConfig: - table: dict[str, object] = ( - {str(key): item for key, item in value.items()} if isinstance(value, dict) else {} - ) - return RulesCacheConfig(enabled=bool(table.get("enabled", True))) - - -def _sql_tests(value: object) -> SqlTestRulesConfig: - table: dict[str, object] = ( - {str(key): item for key, item in value.items()} if isinstance(value, dict) else {} - ) - return SqlTestRulesConfig(pipeline_directory=str(table.get("pipeline_directory", "pipelines"))) - - -def _layout(value: object) -> LayoutConfig: - table: dict[str, object] = ( - {str(key): item for key, item in value.items()} if isinstance(value, dict) else {} - ) - levels: tuple[str, ...] = _strings(table.get("levels")) - return LayoutConfig( - levels=levels or LayoutConfig().levels, - domain_roots=_strings(table.get("domain_roots")), - ) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py new file mode 100644 index 0000000000..4cdbe8af63 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py @@ -0,0 +1,180 @@ +"""Dispatch custom rules to their worker thread beside native built-in rule evaluation.""" + +from __future__ import annotations + +from collections.abc import Callable +from concurrent.futures import Executor, Future, ThreadPoolExecutor +from functools import partial +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity +from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached +from sqlbuild.rule_engine._helpers.engine.native import ( + custom_rule_payloads, + finalize_native_findings, + rule_selected, + selected_codes, +) +from sqlbuild.rule_engine._helpers.run.native_rows import ( + build_rules_request, + evaluate_rules_request, +) +from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES +from sqlbuild.rule_engine.models import ( + CustomRulesOutcome, + Finding, + NativeRulesEvaluation, + Rule, + RulesConfig, + RulesResult, +) + + +def evaluate_native( + *, + project: CompiledProjectView, + config: RulesConfig, + project_dir: Path, + catalogue: tuple[Rule, ...], + dialect: str = "generic", + initial_findings: tuple[Finding, ...] = (), + defer_suppressions: bool = False, + verify_determinism: bool = False, + custom_payloads: list[dict[str, object]] | None = None, + custom_outcome: Future[CustomRulesOutcome] | None = None, +) -> RulesResult: + """Evaluate built-in rules natively while custom rules run incrementally beside them.""" + + payloads: list[dict[str, object]] = ( + custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) + if custom_payloads is None + else custom_payloads + ) + return _evaluate_native_rows( + project=project, + config=config, + project_dir=project_dir, + dialect=dialect, + initial_findings=initial_findings, + defer_suppressions=defer_suppressions, + custom_payloads=payloads, + start_custom=( + (lambda **_: custom_outcome) + if custom_payloads is not None + else partial( + start_custom_rules, + project=project, + config=config, + project_dir=project_dir, + catalogue=catalogue, + custom_payloads=payloads, + dialect=dialect, + verify_determinism=verify_determinism, + ) + ), + ) + + +def _evaluate_native_rows( + *, + project: CompiledProjectView, + config: RulesConfig, + project_dir: Path, + dialect: str, + initial_findings: tuple[Finding, ...], + defer_suppressions: bool, + custom_payloads: list[dict[str, object]], + start_custom: Callable[..., Future[CustomRulesOutcome] | None], +) -> RulesResult: + request: _native.NativeRulesRequest = build_rules_request( + project=project, + config=config, + project_dir=project_dir, + dialect=dialect, + initial_findings=initial_findings, + custom_payloads=custom_payloads, + include_type_proof=any( + rule_selected(config=config, code=code) for code in TYPE_PROOF_RULE_CODES + ), + ) + with ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as pool: + custom_future: Future[CustomRulesOutcome] | None = start_custom(executor=pool) + native: NativeRulesEvaluation = evaluate_rules_request(request) + del request + custom: CustomRulesOutcome = ( + CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) + if custom_future is None + else custom_future.result() + ) + findings: tuple[Finding, ...] = ( + *native.findings, + *custom.findings, + ) + if not defer_suppressions: + findings = finalize_native_findings( + config=config, + project_dir=project_dir, + evaluated_codes=native.selected_codes, + findings=findings, + ) + return RulesResult( + findings=findings, + evaluated_models=native.evaluated_models, + cache_hits=(native.cache_hits + native.cache_misses if native.reused else native.cache_hits) + + custom.cache_hits, + cache_misses=(0 if native.reused else native.cache_misses) + custom.cache_misses, + built_in_ms=0 if native.reused else native.built_in_ms, + custom_ms=custom.custom_ms, + custom_cpu_ms=custom.custom_cpu_ms, + ) + + +def start_custom_rules( + *, + executor: Executor, + project: CompiledProjectView, + config: RulesConfig, + project_dir: Path, + catalogue: tuple[Rule, ...], + custom_payloads: list[dict[str, object]], + dialect: str, + verify_determinism: bool = False, +) -> Future[CustomRulesOutcome] | None: + """Submit selected custom rules after digesting installed code here, clear of GIL contention.""" + + custom_rules: tuple[Rule, ...] = _selected_custom_rules( + config=config, catalogue=catalogue, custom_payloads=custom_payloads + ) + if not custom_rules: + return None + if config.cache.enabled: + _ = compiled_code_identity() + return executor.submit( + evaluate_custom_rules_cached, + project=project, + config=config, + project_dir=project_dir.resolve(), + rules=custom_rules, + dialect=dialect, + verify_determinism=verify_determinism, + implementation_fingerprints={ + str(payload["code"]): str(payload["implementation_fingerprint"]) + for payload in custom_payloads + }, + ) + + +def _selected_custom_rules( + *, + config: RulesConfig, + catalogue: tuple[Rule, ...], + custom_payloads: list[dict[str, object]], +) -> tuple[Rule, ...]: + if not custom_payloads: + return () + selected: frozenset[str] = frozenset( + selected_codes(config=config, custom_payloads=custom_payloads) + ) + return tuple(rule for rule in catalogue if rule.custom and rule.code in selected) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index fd3d9d5d8e..140c20c2cd 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -2,19 +2,17 @@ from __future__ import annotations -import hashlib import os -import threading import time from collections.abc import Callable -from dataclasses import asdict, dataclass, replace +from dataclasses import replace from pathlib import Path from typing import Any -import orjson - from sqlbuild._native import ( + CustomRuleCache, compiled_project_record, + custom_rule_identity, custom_rule_implementation_fingerprint, custom_rule_import_closure, ) @@ -23,50 +21,17 @@ from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine.fact_replay import full_fact_keys, is_tree_fact -from sqlbuild.rule_engine._helpers.host.custom_host_pool import ( - decode_readset, - finding_payload, - run_custom_hosts, -) +from sqlbuild.rule_engine._helpers.host.custom_host_pool import run_custom_hosts from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.classes.fact_digests import FactDigests from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.rule_engine.constants import ( CUSTOM_HOST_RUNTIME_VERSION, - CUSTOM_RULE_PROJECT_SUBJECT, CUSTOM_RULES_CACHE_FILE, - CUSTOM_RULES_CACHE_VERSION, -) -from sqlbuild.rule_engine.exceptions import FactDigestError, RulesError -from sqlbuild.rule_engine.models import ( - CustomHostEvaluation, - CustomHostRun, - CustomRulesOutcome, - Rule, - RulesConfig, ) -from sqlbuild.rule_engine.types import FactKey, RuleSubject - - -@dataclass(frozen=True) -class _Subject: - key: str - identity: str - - -@dataclass(frozen=True) -class _CachedEntry: - subject_identity: str - readset: int - reads_digest: str - findings: tuple[dict[str, object], ...] - - -@dataclass -class _RuleCache: - identity: str - readsets: list[tuple[FactKey, ...]] - entries: dict[str, _CachedEntry] +from sqlbuild.rule_engine.exceptions import FactDigestError +from sqlbuild.rule_engine.models import CustomHostRun, CustomRulesOutcome, Rule, RulesConfig +from sqlbuild.rule_engine.types import CustomEvaluationRow, FactKey def evaluate_custom_rules_cached( @@ -87,75 +52,43 @@ def evaluate_custom_rules_cached( if not custom: return CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) facts_project: CompiledProjectView = host_project(project) - if not config.cache.enabled: - uncached: CustomHostRun = run_custom_hosts( - project=facts_project, - config=config, - project_dir=project_dir, - dialect=dialect, - rules=custom, - model_paths=_model_paths(project), - plan={rule.code: None for rule in custom}, - track_reads=False, - verify_determinism=verify_determinism, - ) - return CustomRulesOutcome( - findings=_ordered_findings( - rules=custom, - subjects=_subjects(project=project, rules=custom), - results={key: value.findings for key, value in uncached.evaluated.items()}, - ), - cache_hits=0, - cache_misses=len(custom), - custom_ms=_elapsed_ms(started), - custom_cpu_ms=_cpu_ms(started_cpu), - ) + enabled: bool = config.cache.enabled + cache: CustomRuleCache = CustomRuleCache( + path=( + rules_bulk_cache_path(project_dir=project_dir, file_name=CUSTOM_RULES_CACHE_FILE) + if enabled + else None + ), + rules=[ + ( + rule.code, + _rule_identity( + rule=rule, + config=config, + project_dir=project_dir, + dialect=dialect, + fingerprint=(implementation_fingerprints or {}).get(rule.code), + ) + if enabled + else None, + None if rule.subject is None else rule.subject.value, + ) + for rule in custom + ], + target_name=project.effective_target_name, + models=[ + (model.relative_path.as_posix(), model.name, _materialization(model)) + for model in project.models + ], + ) digests: FactDigests = FactDigests( views=_views_factory(project=facts_project, project_dir=project_dir, dialect=dialect) ) - subjects: dict[str, tuple[_Subject, ...]] = _subjects(project=project, rules=custom) - identities: dict[str, str] = { - rule.code: _rule_identity( - rule=rule, - config=config, - project_dir=project_dir, - dialect=dialect, - fingerprint=(implementation_fingerprints or {}).get(rule.code), - ) - for rule in custom - } - cache_path: Path = rules_bulk_cache_path( - project_dir=project_dir, file_name=CUSTOM_RULES_CACHE_FILE - ) - stored: dict[str, _RuleCache] = _read_cache(cache_path) - results: dict[tuple[str, str], tuple[dict[str, object], ...]] = {} - plan: dict[str, list[str] | None] = {} - hits: int = 0 - for rule in custom: - cached: _RuleCache | None = stored.get(rule.code) - if cached is not None and cached.identity != identities[rule.code]: - cached = None - current_digests: dict[int, str | None] = {} - missing: list[str] = [] - for subject in subjects[rule.code]: - entry: _CachedEntry | None = None if cached is None else cached.entries.get(subject.key) - if ( - entry is not None - and cached is not None - and entry.subject_identity == subject.identity - and _readset_digest( - cached=cached, index=entry.readset, digests=digests, memo=current_digests - ) - == entry.reads_digest - ): - results[(rule.code, subject.key)] = entry.findings - hits += 1 - else: - missing.append(subject.key) - if missing: - plan[rule.code] = None if len(missing) == len(subjects[rule.code]) else sorted(missing) - misses: int = sum( - len(subjects[code]) if planned is None else len(planned) for code, planned in plan.items() + plan: dict[str, list[str] | None] = cache.plan( + digests=[ + _readset_digest(readset=readset, digests=digests) + for readset in cache.pending_readsets() + ] ) if plan: run: CustomHostRun = run_custom_hosts( @@ -164,30 +97,16 @@ def evaluate_custom_rules_cached( project_dir=project_dir, dialect=dialect, rules=custom, - model_paths=_model_paths(project), + model_paths=tuple(sorted(model.relative_path.as_posix() for model in project.models)), plan=plan, - track_reads=True, + track_reads=enabled, verify_determinism=verify_determinism, ) - _validate_paths(project=project, rules=custom, evaluated=run.evaluated) - stored = _store( - stored=stored, - identities=identities, - subjects=subjects, - run=run, - digests=digests, - ) - for key, value in run.evaluated.items(): - results[key] = value.findings - _write_cache(path=cache_path, stored=stored, subjects=subjects) + cache.record(evaluated=_evaluations(run=run, digests=digests, cacheable=enabled)) return CustomRulesOutcome( - findings=_ordered_findings( - rules=custom, - subjects=subjects, - results=results, - ), - cache_hits=hits, - cache_misses=misses, + findings=tuple(cache.findings()), + cache_hits=cache.hits, + cache_misses=cache.misses, custom_ms=_elapsed_ms(started), custom_cpu_ms=_cpu_ms(started_cpu), ) @@ -202,38 +121,6 @@ def build() -> RuleFactViews: return build -def _subjects( - *, project: CompiledProjectView, rules: tuple[Rule, ...] -) -> dict[str, tuple[_Subject, ...]]: - project_subject: tuple[_Subject, ...] = ( - _Subject( - key=CUSTOM_RULE_PROJECT_SUBJECT, - identity=_digest([len(project.models), project.effective_target_name]), - ), - ) - model_subjects: tuple[_Subject, ...] = ( - tuple( - _Subject( - key=model.relative_path.as_posix(), - identity=_digest( - [model.name, model.relative_path.as_posix(), _materialization(model)] - ), - ) - for model in sorted(project.models, key=lambda item: item.relative_path.as_posix()) - ) - if any(rule.subject is not RuleSubject.PROJECT for rule in rules) - else () - ) - return { - rule.code: project_subject if rule.subject is RuleSubject.PROJECT else model_subjects - for rule in rules - } - - -def _model_paths(project: CompiledProjectView) -> tuple[str, ...]: - return tuple(sorted(model.relative_path.as_posix() for model in project.models)) - - def _materialization(model: Any) -> str | None: value: object = model.config.values.get("materialized") return value if isinstance(value, str) else None @@ -242,44 +129,33 @@ def _materialization(model: Any) -> str | None: def _rule_identity( *, rule: Rule, config: RulesConfig, project_dir: Path, dialect: str, fingerprint: str | None ) -> str: - return _digest( - [ - CUSTOM_RULES_CACHE_VERSION, - CUSTOM_HOST_RUNTIME_VERSION, - compiled_code_identity(), - rule.code, - rule.family, - rule.slug, - rule.message, - rule.remediation, - None if rule.subject is None else rule.subject.value, - fingerprint - if fingerprint is not None - else custom_rule_implementation_fingerprint( - rule=rule, - project_dir=project_dir, - import_closure=custom_rule_import_closure(rule=rule, project_dir=project_dir), - ), - asdict(config), - dialect, - str(project_dir.resolve()), - ] + return custom_rule_identity( + code_identity=compiled_code_identity(), + runtime_version=CUSTOM_HOST_RUNTIME_VERSION, + code=rule.code, + family=rule.family, + slug=rule.slug, + message=rule.message, + remediation=rule.remediation, + subject=None if rule.subject is None else rule.subject.value, + implementation_fingerprint=fingerprint + if fingerprint is not None + else custom_rule_implementation_fingerprint( + rule=rule, + project_dir=project_dir, + import_closure=custom_rule_import_closure(rule=rule, project_dir=project_dir), + ), + config=config, + dialect=dialect, + project_dir=str(project_dir.resolve()), ) -def _readset_digest( - *, cached: _RuleCache, index: int, digests: FactDigests, memo: dict[int, str | None] -) -> str | None: - if index not in memo: - try: - memo[index] = ( - digests.combined(cached.readsets[index]) - if 0 <= index < len(cached.readsets) - else None - ) - except FactDigestError: - memo[index] = None - return memo[index] +def _readset_digest(*, readset: tuple[FactKey, ...], digests: FactDigests) -> str | None: + try: + return digests.combined(readset) + except FactDigestError: + return None def host_project(project: CompiledProjectView) -> CompiledProject: @@ -302,91 +178,30 @@ def host_project(project: CompiledProjectView) -> CompiledProject: ) -def _validate_paths( - *, - project: CompiledProjectView, - rules: tuple[Rule, ...], - evaluated: dict[tuple[str, str], CustomHostEvaluation], -) -> None: - codes: frozenset[str] = frozenset(rule.code for rule in rules) - model_rules: frozenset[str] = frozenset( - rule.code for rule in rules if rule.subject is RuleSubject.MODEL - ) - model_paths: frozenset[str] = frozenset( - model.relative_path.as_posix() for model in project.models - ) - for (code, _subject), result in evaluated.items(): - for finding in result.findings: - if finding.get("code") not in codes: - raise RulesError( - f"custom rule host returned a finding for unselected rule {finding.get('code')}" - ) - if code in model_rules and finding.get("path") not in model_paths: - raise RulesError( - "model-subject custom rule returned a finding for unrelated path " - f"{finding.get('path')}" - ) - +def _evaluations( + *, run: CustomHostRun, digests: FactDigests, cacheable: bool +) -> list[CustomEvaluationRow]: + """Each evaluated invocation with its read set and its digest when it may be cached.""" -def _store( - *, - stored: dict[str, _RuleCache], - identities: dict[str, str], - subjects: dict[str, tuple[_Subject, ...]], - run: CustomHostRun, - digests: FactDigests, -) -> dict[str, _RuleCache]: - updated: dict[str, _RuleCache] = {} full_keys: tuple[FactKey, ...] | None = None - for code, identity in identities.items(): - previous: _RuleCache | None = stored.get(code) - current: _RuleCache = ( - _RuleCache(identity=identity, readsets=[], entries={}) - if previous is None or previous.identity != identity - else _RuleCache( - identity=identity, - readsets=list(previous.readsets), - entries=dict(previous.entries), - ) - ) - updated[code] = current - by_subject: dict[str, _Subject] = {subject.key: subject for subject in subjects[code]} - positions: dict[tuple[FactKey, ...], int] = { - readset: index for index, readset in enumerate(current.readsets) - } - combined_by_readset: dict[tuple[FactKey, ...], str | None] = {} - for (evaluated_code, subject_key), result in run.evaluated.items(): - if evaluated_code != code: - continue - if code in run.uncacheable: - current.entries.pop(subject_key, None) - continue + combined: dict[tuple[FactKey, ...], str | None] = {} + rows: list[CustomEvaluationRow] = [] + for (code, subject), result in run.evaluated.items(): + stored: tuple[tuple[FactKey, ...], str | None] | None = None + if cacheable and code not in run.uncacheable: reads: tuple[FactKey, ...] | None = result.reads if reads is None: if full_keys is None: full_keys = full_fact_keys(digests.views) reads = (*run.untracked_reads.get(code, ()), *full_keys) readset: tuple[FactKey, ...] = tuple(sorted(set(reads))) - if readset not in combined_by_readset: - combined_by_readset[readset] = _observed_digest( + if readset not in combined: + combined[readset] = _observed_digest( readset=readset, observed=run.observed, digests=digests ) - combined: str | None = combined_by_readset[readset] - if combined is None: - current.entries.pop(subject_key, None) - continue - index: int | None = positions.get(readset) - if index is None: - index = len(current.readsets) - current.readsets.append(readset) - positions[readset] = index - current.entries[subject_key] = _CachedEntry( - subject_identity=by_subject[subject_key].identity, - readset=index, - reads_digest=combined, - findings=result.findings, - ) - return updated + stored = (readset, combined[readset]) + rows.append((code, subject, stored, list(result.findings))) + return rows def _observed_digest( @@ -403,90 +218,6 @@ def _observed_digest( return None -def _ordered_findings( - *, - rules: tuple[Rule, ...], - subjects: dict[str, tuple[_Subject, ...]], - results: dict[tuple[str, str], tuple[dict[str, object], ...]], -) -> tuple[dict[str, object], ...]: - findings: list[dict[str, object]] = [] - for rule in rules: - for subject in subjects[rule.code]: - findings.extend(results.get((rule.code, subject.key), ())) - return tuple(findings) - - -def _read_cache(path: Path) -> dict[str, _RuleCache]: - try: - payload: object = orjson.loads(path.read_bytes()) - except (OSError, orjson.JSONDecodeError): - return {} - if not isinstance(payload, dict) or payload.get("version") != CUSTOM_RULES_CACHE_VERSION: - return {} - rules: object = payload.get("rules") - if not isinstance(rules, dict): - return {} - stored: dict[str, _RuleCache] = {} - for code, value in rules.items(): - try: - stored[str(code)] = _RuleCache( - identity=str(value["identity"]), - readsets=list(map(decode_readset, value["readsets"])), - entries={ - str(subject): _decode_entry(entry) - for subject, entry in value["entries"].items() - }, - ) - except (KeyError, TypeError, ValueError, IndexError): - continue - return stored - - -def _write_cache( - *, path: Path, stored: dict[str, _RuleCache], subjects: dict[str, tuple[_Subject, ...]] -) -> None: - rules: dict[str, object] = {} - for code, cached in stored.items(): - if code not in subjects: - continue - live: frozenset[str] = frozenset(subject.key for subject in subjects[code]) - entries: dict[str, _CachedEntry] = { - key: entry for key, entry in cached.entries.items() if key in live - } - used: list[int] = sorted({entry.readset for entry in entries.values()}) - remap: dict[int, int] = {old: new for new, old in enumerate(used)} - rules[code] = { - "identity": cached.identity, - "readsets": [list(map(list, cached.readsets[index])) for index in used], - "entries": { - key: [ - entry.subject_identity, - remap[entry.readset], - entry.reads_digest, - list(entry.findings), - ] - for key, entry in entries.items() - }, - } - path.parent.mkdir(parents=True, exist_ok=True) - temporary: Path = path.with_suffix(f".tmp-{os.getpid()}-{threading.get_ident()}") - temporary.write_bytes(orjson.dumps({"version": CUSTOM_RULES_CACHE_VERSION, "rules": rules})) - temporary.replace(path) - - -def _decode_entry(value: Any) -> _CachedEntry: - return _CachedEntry( - subject_identity=str(value[0]), - readset=int(value[1]), - reads_digest=str(value[2]), - findings=tuple(map(finding_payload, value[3])), - ) - - -def _digest(value: object) -> str: - return hashlib.sha256(orjson.dumps(value, default=str)).hexdigest() - - def _cpu_seconds() -> float: """This thread's CPU plus that of finished child processes, which run the rule hosts.""" diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index d50dc4b254..b1c6147abe 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -4,10 +4,7 @@ import inspect import json -from collections.abc import Callable -from concurrent.futures import Executor, Future, ThreadPoolExecutor -from dataclasses import asdict, replace -from functools import partial +from dataclasses import asdict from pathlib import Path import sqlbuild._native as _native @@ -16,177 +13,20 @@ custom_rule_import_closure, custom_rule_test_evidence, ) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.run.native_rows import ( - build_rules_request, - evaluate_rules_request, finalize_rules_rows, ) from sqlbuild.rule_engine.constants import ( RULES_NATIVE_API_VERSION, - TYPE_PROOF_RULE_CODES, ) from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import ( - CustomRulesOutcome, Finding, - NativeRulesEvaluation, Rule, RulesConfig, - RulesResult, ) -def evaluate_native( - *, - project: CompiledProjectView, - config: RulesConfig, - project_dir: Path, - catalogue: tuple[Rule, ...], - dialect: str = "generic", - initial_findings: tuple[Finding, ...] = (), - defer_suppressions: bool = False, - verify_determinism: bool = False, - custom_payloads: list[dict[str, object]] | None = None, - custom_outcome: Future[CustomRulesOutcome] | None = None, -) -> RulesResult: - """Evaluate built-in rules natively while custom rules run incrementally beside them.""" - - payloads: list[dict[str, object]] = ( - custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) - if custom_payloads is None - else custom_payloads - ) - return _evaluate_native_rows( - project=project, - config=config, - project_dir=project_dir, - dialect=dialect, - initial_findings=initial_findings, - defer_suppressions=defer_suppressions, - custom_payloads=payloads, - start_custom=( - (lambda **_: custom_outcome) - if custom_payloads is not None - else partial( - start_custom_rules, - project=project, - config=config, - project_dir=project_dir, - catalogue=catalogue, - custom_payloads=payloads, - dialect=dialect, - verify_determinism=verify_determinism, - ) - ), - ) - - -def _evaluate_native_rows( - *, - project: CompiledProjectView, - config: RulesConfig, - project_dir: Path, - dialect: str, - initial_findings: tuple[Finding, ...], - defer_suppressions: bool, - custom_payloads: list[dict[str, object]], - start_custom: Callable[..., Future[CustomRulesOutcome] | None], -) -> RulesResult: - request: _native.NativeRulesRequest = build_rules_request( - project=project, - config=config, - project_dir=project_dir, - dialect=dialect, - initial_findings=initial_findings, - custom_payloads=custom_payloads, - include_type_proof=any( - _rule_selected(config=config, code=code) for code in TYPE_PROOF_RULE_CODES - ), - ) - with ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as pool: - custom_future: Future[CustomRulesOutcome] | None = start_custom(executor=pool) - native: NativeRulesEvaluation = evaluate_rules_request(request) - del request - custom: CustomRulesOutcome = ( - CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) - if custom_future is None - else custom_future.result() - ) - findings: tuple[Finding, ...] = ( - *native.findings, - *(decode_rule_finding(value) for value in custom.findings), - ) - if not defer_suppressions: - findings = finalize_native_findings( - config=config, - project_dir=project_dir, - evaluated_codes=native.selected_codes, - findings=findings, - ) - return RulesResult( - findings=findings, - evaluated_models=native.evaluated_models, - cache_hits=(native.cache_hits + native.cache_misses if native.reused else native.cache_hits) - + custom.cache_hits, - cache_misses=(0 if native.reused else native.cache_misses) + custom.cache_misses, - built_in_ms=0 if native.reused else native.built_in_ms, - custom_ms=custom.custom_ms, - custom_cpu_ms=custom.custom_cpu_ms, - ) - - -def start_custom_rules( - *, - executor: Executor, - project: CompiledProjectView, - config: RulesConfig, - project_dir: Path, - catalogue: tuple[Rule, ...], - custom_payloads: list[dict[str, object]], - dialect: str, - verify_determinism: bool = False, -) -> Future[CustomRulesOutcome] | None: - """Submit selected custom rules after digesting installed code here, clear of GIL contention.""" - - custom_rules: tuple[Rule, ...] = _selected_custom_rules( - config=config, catalogue=catalogue, custom_payloads=custom_payloads - ) - if not custom_rules: - return None - if config.cache.enabled: - _ = compiled_code_identity() - return executor.submit( - evaluate_custom_rules_cached, - project=project, - config=config, - project_dir=project_dir.resolve(), - rules=custom_rules, - dialect=dialect, - verify_determinism=verify_determinism, - implementation_fingerprints={ - str(payload["code"]): str(payload["implementation_fingerprint"]) - for payload in custom_payloads - }, - ) - - -def _selected_custom_rules( - *, - config: RulesConfig, - catalogue: tuple[Rule, ...], - custom_payloads: list[dict[str, object]], -) -> tuple[Rule, ...]: - if not custom_payloads: - return () - selected: frozenset[str] = frozenset( - _selected_codes(config=config, custom_payloads=custom_payloads) - ) - return tuple(rule for rule in catalogue if rule.custom and rule.code in selected) - - def finalize_native_findings( *, config: RulesConfig, @@ -196,69 +36,14 @@ def finalize_native_findings( ) -> tuple[Finding, ...]: """Apply exception policy once to completed SQL, native, and custom findings.""" - return _with_fixable( - original=findings, - finalized=finalize_rules_rows( - config=config, - project_dir=project_dir, - evaluated_codes=evaluated_codes, - findings=findings, - ), - ) - - -def _with_fixable( - *, original: tuple[Finding, ...], finalized: tuple[Finding, ...] -) -> tuple[Finding, ...]: - fixable: frozenset[tuple[str, str, int, int, str]] = frozenset( - _finding_identity(finding) for finding in original if finding.fixable - ) - return tuple( - replace(finding, fixable=True) if _finding_identity(finding) in fixable else finding - for finding in finalized - ) - - -def _finding_identity(finding: Finding) -> tuple[str, str, int, int, str]: - return ( - finding.code, - finding.path.as_posix(), - finding.line, - finding.column, - finding.message, + return finalize_rules_rows( + config=config, + project_dir=project_dir, + evaluated_codes=evaluated_codes, + findings=findings, ) -def load_native_config(project_dir: Path) -> dict[str, object]: - """Load strict rules TOML through the native configuration owner.""" - - try: - payload: object = json.loads(_native.load_config_json(str(project_dir.resolve()))) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - if not isinstance(payload, dict): - raise RulesError("native rules engine returned invalid configuration") - return {str(key): value for key, value in payload.items()} - - -def native_catalogue() -> tuple[dict[str, object], ...]: - """Return native-owned built-in rule metadata.""" - - try: - payload: object = json.loads(_native.catalogue_json()) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - if not isinstance(payload, dict) or not isinstance(payload.get("rules"), list): - raise RulesError("native rules engine returned an invalid catalogue") - rules: list[object] = payload["rules"] - if any(not isinstance(value, dict) for value in rules): - raise RulesError("native rules engine returned invalid rule metadata") - normalized: list[dict[str, object]] = [] - for value in rules: - normalized.append({str(key): item for key, item in value.items()}) - return tuple(normalized) - - def native_selected_codes( *, config: RulesConfig, @@ -268,7 +53,7 @@ def native_selected_codes( ) -> tuple[str, ...]: """Resolve the active ruleset through the native Fensu adapter.""" - return _selected_codes( + return selected_codes( config=config, custom_payloads=( custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) @@ -278,7 +63,7 @@ def native_selected_codes( ) -def _selected_codes( +def selected_codes( *, config: RulesConfig, custom_payloads: list[dict[str, object]] ) -> tuple[str, ...]: request: dict[str, object] = { @@ -316,7 +101,7 @@ def _config_payload(config: RulesConfig) -> dict[str, object]: return asdict(config) -def _rule_selected(*, config: RulesConfig, code: str) -> bool: +def rule_selected(*, config: RulesConfig, code: str) -> bool: selected: bool = any(code.startswith(selector) for selector in config.select) ignored: bool = any(code.startswith(selector) for selector in config.ignore) return selected and not ignored @@ -390,46 +175,3 @@ def _custom_rule_source_payload(*, rule: Rule, project_dir: Path) -> dict[str, o "source_column": 1, "owner": f"{source or 'rules'}:{check_owner}", } - - -def decode_rule_finding(value: object) -> Finding: - if not isinstance(value, dict): - raise RulesError("native rules engine returned an invalid finding") - payload: dict[str, object] = {str(key): item for key, item in value.items()} - code: object = payload.get("code") - path: object = payload.get("path") - line: object = payload.get("line") - column: object = payload.get("column") - message: object = payload.get("message") - remediation: object = payload.get("remediation") - if ( - not isinstance(code, str) - or not isinstance(path, str) - or not isinstance(line, int) - or not isinstance(column, int) - or not isinstance(message, str) - or not isinstance(remediation, str) - ): - raise RulesError("native rules engine returned an invalid finding") - return Finding( - code=code, - path=Path(path), - line=line, - column=column, - message=message, - remediation=remediation, - unevaluated=payload.get("unevaluated") is True, - fixable=payload.get("fixable") is True, - ) - - -def _finding_payload(finding: Finding) -> dict[str, object]: - return { - "unevaluated": finding.unevaluated, - "code": finding.code, - "path": finding.path.as_posix(), - "line": finding.line, - "column": finding.column, - "message": finding.message, - "remediation": finding.remediation, - } diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index 7096622f1d..2d039faef4 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -10,7 +10,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.rule_engine._helpers.engine.config import load_rules_config from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, @@ -18,6 +17,7 @@ from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.exceptions import RuleAssertionError, RuleUsageError from sqlbuild.rule_engine.main._evaluate import evaluate +from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import ( Rule, RuleCase, @@ -39,7 +39,7 @@ def run_rule_case(*, rule: RuleCheck | Rule, test_case: RuleCase) -> RuleResult: root: Path = Path(temporary) _ = _write_rule_project(root=root, rule=resolved, test_case=test_case) project: CompiledProjectView = _compile_rule_project(root=root) - config: RulesConfig = load_rules_config(root) + config: RulesConfig = load_rules_config(project_dir=root) result: RulesResult = evaluate( project=project, config=config, diff --git a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py b/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py index 5ff0195d61..1b1625c074 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py @@ -13,8 +13,8 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors +from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine._helpers.engine.native import evaluate_native from sqlbuild.rule_engine.models import ResolvedRuleset, Rule, RulesConfig, RulesResult diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py index 5f67c8da7e..843d59146f 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py @@ -9,18 +9,8 @@ import orjson import sqlbuild._native as _native -from sqlbuild._native import ( - compiled_project_facts, - declaration_rows, - plain_typed_value, -) +from sqlbuild._native import compiled_project_facts, declaration_rows from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.compile.models import ( - CompiledModelSqlTestPayload, - CompiledSqlScenario, - CompiledSqlTest, - CompileSqlTestCte, -) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection @@ -32,7 +22,6 @@ ) from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import Finding, NativeRulesEvaluation, RulesConfig -from sqlbuild.rule_engine.types import FindingRow def build_rules_request( @@ -85,7 +74,7 @@ def build_rules_request( ], "public_enums": public_enums, "public_constants": public_constants, - "initial_findings": [finding_row(finding) for finding in initial_findings], + "initial_findings": list(initial_findings), "types_equal": partial(types_equal, dialect=dialect), } try: @@ -93,8 +82,8 @@ def build_rules_request( request = _native.build_rules_request( project_rows, (compiled_project_facts(project), [model.name for model in project.models]), - [_sql_test_row(test) for test in project.sql_tests], - [_scenario_row(scenario) for scenario in project.sql_scenarios], + list(project.sql_tests), + list(project.sql_scenarios), ) except (ValueError, TypeError) as error: raise RulesError(str(error)) from error @@ -111,7 +100,7 @@ def evaluate_rules_request(request: _native.NativeRulesRequest) -> NativeRulesEv except (ValueError, TypeError) as error: raise RulesError(str(error)) from error return NativeRulesEvaluation( - findings=tuple(finding_from_row(row) for row in faults), + findings=tuple(faults), selected_codes=tuple(selected_codes), evaluated_models=evaluated, cache_hits=hits, @@ -128,49 +117,20 @@ def finalize_rules_rows( evaluated_codes: tuple[str, ...], findings: tuple[Finding, ...], ) -> tuple[Finding, ...]: - """Apply the exception policy natively to finding rows.""" + """Apply the exception policy natively, keeping each original finding's fixable mark.""" try: - rows: list[FindingRow] = _native.finalize_rule_findings_rows( + finalized: list[Finding] = _native.finalize_rule_findings_rows( { "project_dir": str(project_dir.resolve()), "config_json": _encoded(asdict(config)), "evaluated_codes": list(evaluated_codes), - "findings": [finding_row(finding) for finding in findings], + "findings": list(findings), } ) except (ValueError, TypeError) as error: raise RulesError(str(error)) from error - return tuple(finding_from_row(row) for row in rows) - - -def finding_row(finding: Finding) -> FindingRow: - """One finding as the native boundary's row.""" - - return ( - finding.unevaluated, - finding.code, - finding.path.as_posix(), - finding.line, - finding.column, - finding.message, - finding.remediation, - ) - - -def finding_from_row(row: FindingRow) -> Finding: - """One native finding row as a finding.""" - - unevaluated, code, path, line, column, message, remediation = row - return Finding( - code=code, - path=Path(path), - line=line, - column=column, - message=message, - remediation=remediation, - unevaluated=unevaluated, - ) + return tuple(finalized) def _encoded(value: object) -> bytes: @@ -178,67 +138,3 @@ def _encoded(value: object) -> bytes: return orjson.dumps(value, option=orjson.OPT_SORT_KEYS, default=str) except orjson.JSONEncodeError as error: raise RulesError(str(error)) from error - - -def _sql_test_row(test: CompiledSqlTest) -> tuple[object, ...]: - payload: object = ( - ( - _cte_rows(test.payload.authored_ctes), - _cte_rows(test.payload.expected_ctes), - _cte_rows(test.payload.assertion_ctes), - bool(test.payload.macro_mocks), - bool(test.payload.model_query_overrides), - ) - if isinstance(test.payload, CompiledModelSqlTestPayload) - else None - ) - case: object = ( - None - if test.case_name is None - else ( - test.parent_name, - test.case_name, - test.case_index, - test.case_fingerprint, - [ - (parameter.name, parameter.value_type.value, parameter.nullable) - for parameter in test.parameter_schema - ], - [(name, plain_typed_value(value)) for name, value in test.parameter_values], - ) - ) - return ( - (test.source_path or test.test_file.relative_path).as_posix(), - (test.ownership_root or test.test_file.ownership_root).as_posix(), - test.block_index or test.test_block.test_index, - test.name, - test.explicit_name, - test.mode.value, - ( - list(test.expected_model_names), - list(test.assertion_names), - list(test.assertion_target_model_names), - list(test.target_model_names), - ), - [(resource.kind.value, resource.name) for resource in test.tested_resources], - payload, - case, - ) - - -def _cte_rows(ctes: tuple[CompileSqlTestCte, ...]) -> list[tuple[str, str]]: - return [(cte.name, cte.sql_body) for cte in ctes] - - -def _scenario_row(scenario: CompiledSqlScenario) -> tuple[object, ...]: - description: object | None = scenario.scenario_file.header_values.get("description") - return ( - (scenario.source_path or scenario.scenario_file.relative_path).as_posix(), - (scenario.ownership_root or scenario.scenario_file.ownership_root).as_posix(), - scenario.name, - description if isinstance(description, str) else None, - list(scenario.expected_model_names), - list(scenario.assertion_names), - list(scenario.assertion_target_model_names), - list(scenario.target_model_names), - ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py new file mode 100644 index 0000000000..8ae88128f3 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py @@ -0,0 +1,152 @@ +"""Thread and future handling around one rules run and early SQL lint.""" + +from __future__ import annotations + +from concurrent.futures import Executor, Future, ThreadPoolExecutor +from contextvars import Token +from pathlib import Path +from threading import Event + +from sqlbuild.compiler.compile.models import ( + CompiledObjectKey, + CompiledSqlExpansion, + DeclarationScopeBuild, + SqlExpansionContext, +) +from sqlbuild.compiler.compile.types import CompileInputsView +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.lint.classes.stop_context import LintStopContext +from sqlbuild.lint.main.build_expansion_context import build_expansion_context +from sqlbuild.lint.main.run_lint import run_lint +from sqlbuild.lint.models import LintConfig, LintRunResult +from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native, start_custom_rules +from sqlbuild.rule_engine._helpers.run.rules import ( + early_lint_expansions, + finish_rule_run, + plan_rule_run, + run_planned_sql_rules, + sql_lint_preparation, +) +from sqlbuild.rule_engine.models import ( + CustomRulesOutcome, + PreparedSqlLint, + PreparedSqlLintResult, + RuleRunPlan, + RulesConfig, + RulesResult, + RulesRunResult, + SqlExpansionReuse, + SqlRulesEvaluation, +) + + +def evaluate_rules( + *, + graph: ProjectGraph, + discovered_inputs: DiscoveredProjectInputs, + config: RulesConfig, + project_dir: Path, + dialect: str, + selected_keys: frozenset[CompiledObjectKey] | None = None, + prepared_sql: PreparedSqlLint | None = None, + expansion_reuse: SqlExpansionReuse | None = None, + no_sql_analysis: bool = False, +) -> RulesRunResult: + """Evaluate independent rule phases concurrently, then finalize their combined findings.""" + + plan: RuleRunPlan = plan_rule_run( + graph=graph, + config=config, + project_dir=project_dir, + selected_keys=selected_keys, + no_sql_analysis=no_sql_analysis, + ) + with ( + ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as custom, + ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-rules") as executor, + ): + custom_outcome: Future[CustomRulesOutcome] | None = start_custom_rules( + executor=custom, + project=plan.project, + config=plan.config, + project_dir=plan.project_dir, + catalogue=plan.catalogue, + custom_payloads=plan.custom_payloads, + dialect=dialect, + ) + native_result: Future[RulesResult] = executor.submit( + evaluate_native, + project=plan.project, + config=plan.config, + project_dir=plan.project_dir, + catalogue=plan.catalogue, + dialect=dialect, + defer_suppressions=True, + custom_payloads=plan.custom_payloads, + custom_outcome=custom_outcome, + ) + sql_result: SqlRulesEvaluation = run_planned_sql_rules( + plan=plan, + graph=graph, + discovered_inputs=discovered_inputs, + dialect=dialect, + prepared_sql=prepared_sql, + expansion_reuse=expansion_reuse, + ) + result: RulesResult = native_result.result() + return finish_rule_run(plan=plan, sql_result=sql_result, result=result) + + +def prepare_sql_rules( + *, inputs: CompileInputsView, executor: Executor, dialect: str, stop: Event +) -> PreparedSqlLint | None: + """Start early SQL lint on `executor` when the project selects SQL rules without a cache.""" + + preparation: tuple[Path, tuple[str, ...], LintConfig] | None = sql_lint_preparation( + inputs=inputs, dialect=dialect + ) + if preparation is None: + return None + project_dir, codes, config = preparation + return PreparedSqlLint( + codes=codes, + project_dir=project_dir, + future=executor.submit( + _prepare_sql_lint, + project_dir=project_dir, + config=config, + discovered_inputs=inputs.discovered_inputs, + static_declaration_scope=inputs.declaration_scope, + compiled_expansions=early_lint_expansions(inputs.model_inputs), + stop=stop, + ), + ) + + +def _prepare_sql_lint( + *, + project_dir: Path, + config: LintConfig, + discovered_inputs: DiscoveredProjectInputs, + static_declaration_scope: DeclarationScopeBuild | None, + compiled_expansions: dict[Path, CompiledSqlExpansion], + stop: Event, +) -> PreparedSqlLintResult: + stop_token: Token[Event | None] = LintStopContext.active.set(stop) + try: + context: SqlExpansionContext = build_expansion_context( + project_dir=project_dir, + discovered_inputs=discovered_inputs, + static_declaration_scope=static_declaration_scope, + ) + result: LintRunResult = run_lint( + project_dir=project_dir, + config=config, + discovered_inputs=discovered_inputs, + compiled_expansions=compiled_expansions, + expansion_context=context, + ) + finally: + LintStopContext.active.reset(stop_token) + return PreparedSqlLintResult(result=result, context=context) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index cb50216345..9d08af870c 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -2,23 +2,23 @@ from __future__ import annotations -import hashlib import json -import os import time -from concurrent.futures import Executor, Future, ThreadPoolExecutor -from contextvars import Token from dataclasses import asdict, dataclass, replace from pathlib import Path -from threading import Event -from typing import Any -import sqlbuild._native as _native from sqlbuild._native import ( + encode_sql_rule_findings, + load_sql_rule_cache, + lookup_sql_rule_cache, project_with, + selected_sql_rule_codes, + sql_file_rule_key, + sql_model_rule_keys, + sql_rule_lint_key, with_duplicate_literal_hints, + write_sql_rule_cache, ) -from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlExpansion, @@ -34,14 +34,12 @@ ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.constants import ( DEFAULT_MAX_LITERAL_LENGTH, DEFAULT_MAX_RANKING_ORDER_BY, MAX_LITERAL_LENGTH_THRESHOLD, MAX_RANKING_ORDER_BY_THRESHOLD, SAFE_RULE_FIX_CODES, - TEMPLATE_INTERPOLATION_START, ) from sqlbuild.lint.exceptions import NativeLintError from sqlbuild.lint.main.build_expansion_context import build_expansion_context @@ -56,11 +54,7 @@ from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules from sqlbuild.rule_engine._helpers.engine.native import ( custom_rule_payloads, - decode_rule_finding, - evaluate_native, finalize_native_findings, - native_catalogue, - start_custom_rules, ) from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings @@ -69,19 +63,18 @@ from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import ( - CustomRulesOutcome, Finding, PreparedSqlLint, PreparedSqlLintResult, Rule, + RuleRunPlan, RulesConfig, RulesResult, RulesRunResult, SqlExpansionReuse, + SqlRulesEvaluation, ) -_SQL_RULE_CACHE_VERSION: str = "sql-rules-v5" -_RULES_BUILD_IDENTITY: str = _native.BUILD_IDENTITY _SQL_RULE_SUPPRESSION_CODE: str = "SQBRSQL000" @@ -91,31 +84,20 @@ class _SqlRuleInputs: project_files: dict[Path, str] | None -@dataclass(frozen=True) -class _SqlRulesEvaluation: - findings: tuple[Finding, ...] - cache_hits: int - cache_misses: int - - -def evaluate_rules( +def plan_rule_run( *, graph: ProjectGraph, - discovered_inputs: DiscoveredProjectInputs, config: RulesConfig, project_dir: Path, - dialect: str, - selected_keys: frozenset[CompiledObjectKey] | None = None, - prepared_sql: PreparedSqlLint | None = None, - expansion_reuse: SqlExpansionReuse | None = None, - no_sql_analysis: bool = False, -) -> RulesRunResult: - """Evaluate independent rule phases concurrently, then finalize their combined findings.""" + selected_keys: frozenset[CompiledObjectKey] | None, + no_sql_analysis: bool, +) -> RuleRunPlan: + """Build the catalogue and select the rules, project slice and settings one run evaluates.""" + release_freed_memory() effective_config: RulesConfig = resolve_rule_ignore_selectors( config=config, project=graph.project ) - resolved_project_dir: Path = project_dir.resolve() include_custom: bool = _config_references_custom_rules(effective_config) catalogue: tuple[Rule, ...] = build_catalogue( @@ -138,7 +120,6 @@ def evaluate_rules( rule.code for rule in selected if rule.code in TYPE_PROOF_RULE_CODES ) selected = tuple(rule for rule in selected if rule.code not in TYPE_PROOF_RULE_CODES) - native_rules: tuple[Rule, ...] = tuple(rule for rule in selected if not rule.custom) custom_rules: tuple[Rule, ...] = tuple(rule for rule in selected if rule.custom) if custom_rules: _ = verify_custom_rules(rules=custom_rules, project_dir=resolved_project_dir) @@ -155,64 +136,71 @@ def evaluate_rules( select=tuple(rule.code for rule in selected), ignore=(), ) - with ( - ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as custom, - ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-rules") as executor, - ): - custom_outcome: Future[CustomRulesOutcome] | None = start_custom_rules( - executor=custom, - project=selected_project, - config=selected_config, - project_dir=resolved_project_dir, - catalogue=catalogue, - custom_payloads=custom_payloads, - dialect=dialect, - ) - native_result: Future[RulesResult] = executor.submit( - evaluate_native, - project=selected_project, - config=selected_config, - project_dir=resolved_project_dir, - catalogue=catalogue, - dialect=dialect, - defer_suppressions=True, - custom_payloads=custom_payloads, - custom_outcome=custom_outcome, - ) - lint_config: LintConfig = sql_lint_config( - dialect=dialect, - codes=_sql_rule_codes(native_rules), - thresholds=effective_config.thresholds, - relation_keys=compiled_relation_keys(graph.project), - allow_suppressions=effective_config.allow_exceptions, - ) - sql_rule_inputs: _SqlRuleInputs = _prepare_sql_rule_inputs( - rules=native_rules, - project=selected_project, - dialect=_lint_identity(lint_config), - project_dir=resolved_project_dir, - collect_files=prepared_sql is None and model_paths is None, - discovered_inputs=discovered_inputs, - ) - sql_started: float = time.monotonic() - sql_result: _SqlRulesEvaluation = _run_sql_rules( - rules=native_rules, - project_dir=resolved_project_dir, - discovered_inputs=discovered_inputs, - project=selected_project, - lint_config=lint_config, - selected_model_paths=model_paths, - cache_enabled=effective_config.cache.enabled, - prepared_sql=prepared_sql, - expansion_reuse=expansion_reuse, - prepared_inputs=sql_rule_inputs, - ) - sql_ms: int = round((time.monotonic() - sql_started) * 1000) - result: RulesResult = native_result.result() - finalized: tuple[Finding, ...] = finalize_native_findings( - config=selected_config, + return RuleRunPlan( + effective_config=effective_config, project_dir=resolved_project_dir, - evaluated_codes=tuple(rule.code for rule in selected), + catalogue=catalogue, + custom_payloads=custom_payloads, + selected=selected, + skipped_type_proof_rules=skipped_type_proof_rules, + model_paths=model_paths, + project=selected_project, + config=selected_config, + ) + + +def run_planned_sql_rules( + *, + plan: RuleRunPlan, + graph: ProjectGraph, + discovered_inputs: DiscoveredProjectInputs, + dialect: str, + prepared_sql: PreparedSqlLint | None, + expansion_reuse: SqlExpansionReuse | None, +) -> SqlRulesEvaluation: + """Run the planned SQL rules on the caller while built-in rules evaluate natively.""" + + native_rules: tuple[Rule, ...] = tuple(rule for rule in plan.selected if not rule.custom) + lint_config: LintConfig = sql_lint_config( + dialect=dialect, + codes=_sql_rule_codes(native_rules), + thresholds=plan.effective_config.thresholds, + relation_keys=compiled_relation_keys(graph.project), + allow_suppressions=plan.effective_config.allow_exceptions, + ) + sql_rule_inputs: _SqlRuleInputs = _prepare_sql_rule_inputs( + rules=native_rules, + project=plan.project, + dialect=sql_rule_lint_key(config=lint_config), + project_dir=plan.project_dir, + collect_files=prepared_sql is None and plan.model_paths is None, + discovered_inputs=discovered_inputs, + ) + sql_started: float = time.monotonic() + sql_result: SqlRulesEvaluation = _run_sql_rules( + rules=native_rules, + project_dir=plan.project_dir, + discovered_inputs=discovered_inputs, + project=plan.project, + lint_config=lint_config, + selected_model_paths=plan.model_paths, + cache_enabled=plan.effective_config.cache.enabled, + prepared_sql=prepared_sql, + expansion_reuse=expansion_reuse, + prepared_inputs=sql_rule_inputs, + ) + return replace(sql_result, sql_ms=round((time.monotonic() - sql_started) * 1000)) + + +def finish_rule_run( + *, plan: RuleRunPlan, sql_result: SqlRulesEvaluation, result: RulesResult +) -> RulesRunResult: + """Finalize the combined SQL, built-in and custom findings of one run.""" + + finalized: tuple[Finding, ...] = finalize_native_findings( + config=plan.config, + project_dir=plan.project_dir, + evaluated_codes=tuple(rule.code for rule in plan.selected), findings=(*sql_result.findings, *result.findings), ) findings: tuple[Finding, ...] = tuple( @@ -228,16 +216,16 @@ def evaluate_rules( result.evaluated_models - len( {item.path for item in findings if item.unevaluated} - & {model.relative_path for model in selected_project.models} + & {model.relative_path for model in plan.project.models} ), ), unevaluated_resources=len({item.path for item in findings if item.unevaluated}), - built_in_ms=sql_ms + result.built_in_ms, + built_in_ms=sql_result.sql_ms + result.built_in_ms, custom_ms=result.custom_ms, custom_cpu_ms=result.custom_cpu_ms, cache_hits=result.cache_hits + sql_result.cache_hits, cache_misses=result.cache_misses + sql_result.cache_misses, - skipped_type_proof_rules=skipped_type_proof_rules, + skipped_type_proof_rules=plan.skipped_type_proof_rules, ) @@ -281,9 +269,11 @@ def _config_references_custom_rules(config: RulesConfig) -> bool: return any(selector.startswith("XSQBR") for selector in selectors) -def prepare_sql_rules( - *, inputs: CompileInputsView, executor: Executor, dialect: str, stop: Event -) -> PreparedSqlLint | None: +def sql_lint_preparation( + *, inputs: CompileInputsView, dialect: str +) -> tuple[Path, tuple[str, ...], LintConfig] | None: + """The project, SQL rule codes and lint settings early SQL lint runs with, if it should run.""" + project_dir: Path | None = inputs.discovered_inputs.project_dir if project_dir is None or any(diagnostic.is_error for diagnostic in inputs.diagnostics): return None @@ -296,27 +286,19 @@ def prepare_sql_rules( codes: tuple[str, ...] = _selected_sql_codes(config) if not codes: return None - return PreparedSqlLint( - codes=codes, - project_dir=project_dir, - future=executor.submit( - _prepare_sql_lint, - project_dir=project_dir, - config=sql_lint_config( - dialect=dialect, - codes=codes, - thresholds=config.thresholds, - allow_suppressions=config.allow_exceptions, - relation_keys=declared_relation_keys( - model_rows=(model.relation_key_row for model in inputs.model_inputs), - seeds=(seed.schema_entry for seed in inputs.seed_inputs), - sources=(source.source_entry for source in inputs.source_inputs), - ), + return ( + project_dir, + codes, + sql_lint_config( + dialect=dialect, + codes=codes, + thresholds=config.thresholds, + allow_suppressions=config.allow_exceptions, + relation_keys=declared_relation_keys( + model_rows=(model.relation_key_row for model in inputs.model_inputs), + seeds=(seed.schema_entry for seed in inputs.seed_inputs), + sources=(source.source_entry for source in inputs.source_inputs), ), - discovered_inputs=inputs.discovered_inputs, - static_declaration_scope=inputs.declaration_scope, - compiled_expansions=early_lint_expansions(inputs.model_inputs), - stop=stop, ), ) @@ -334,34 +316,6 @@ def early_lint_expansions( return expansions -def _prepare_sql_lint( - *, - project_dir: Path, - config: LintConfig, - discovered_inputs: DiscoveredProjectInputs, - static_declaration_scope: DeclarationScopeBuild | None, - compiled_expansions: dict[Path, CompiledSqlExpansion], - stop: Event, -) -> PreparedSqlLintResult: - stop_token: Token[Event | None] = LintStopContext.active.set(stop) - try: - context: SqlExpansionContext = build_expansion_context( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - static_declaration_scope=static_declaration_scope, - ) - result: LintRunResult = run_lint( - project_dir=project_dir, - config=config, - discovered_inputs=discovered_inputs, - compiled_expansions=compiled_expansions, - expansion_context=context, - ) - finally: - LintStopContext.active.reset(stop_token) - return PreparedSqlLintResult(result=result, context=context) - - def _selection_rules_config( *, config: RulesConfig, @@ -385,16 +339,7 @@ def _selection_rules_config( def _selected_sql_codes(config: RulesConfig) -> tuple[str, ...]: - codes: list[str] = [] - for rule in native_catalogue(): - code: str = str(rule["code"]) - if not code.startswith("SQBRSQL"): - continue - selected: bool = any(code.startswith(selector) for selector in config.select) - ignored: bool = any(code.startswith(selector) for selector in config.ignore) - if selected and not ignored: - codes.append(code) - return tuple(codes) + return tuple(selected_sql_rule_codes(select=list(config.select), ignore=list(config.ignore))) def _selected_project( @@ -491,31 +436,26 @@ def _run_sql_rules( prepared_sql: PreparedSqlLint | None = None, expansion_reuse: SqlExpansionReuse | None = None, prepared_inputs: _SqlRuleInputs | None = None, -) -> _SqlRulesEvaluation: +) -> SqlRulesEvaluation: codes: tuple[str, ...] = _sql_rule_codes(rules) if not codes: - return _SqlRulesEvaluation(findings=(), cache_hits=0, cache_misses=0) + return SqlRulesEvaluation(findings=(), cache_hits=0, cache_misses=0) models_by_path: dict[str, CompiledModelView] = _sql_rule_models(project) - bucket: dict[str, dict[str, object]] = ( - _read_sql_rule_cache(project_dir) if cache_enabled else {} - ) + bucket: dict[str, tuple[str, str]] = _read_sql_rule_cache(project_dir) if cache_enabled else {} identities: dict[str, str] = ( prepared_inputs.identities if prepared_inputs is not None else _sql_model_rule_identities( - rules=rules, project=project, dialect=_lint_identity(lint_config) + rules=rules, project=project, dialect=sql_rule_lint_key(config=lint_config) ) ) findings: list[Finding] = [] - misses: set[str] = set() - for path, identity in identities.items(): - cached: tuple[Finding, ...] | None = _cached_sql_findings( - entry=bucket.get(path), identity=identity - ) - if cached is None: - misses.add(path) - else: - findings.extend(cached) + model_hits: dict[str, tuple[Finding, ...]] = _cached_sql_findings( + bucket=bucket, keys=identities + ) + misses: set[str] = set(identities) - set(model_hits) + for path in identities: + findings.extend(model_hits.get(path, ())) selected_paths: set[Path] = {project_dir / path for path in misses} file_identities: dict[str, str] = {} project_files: dict[Path, str] | None = None @@ -536,25 +476,29 @@ def _run_sql_rules( discovered_inputs=discovered_inputs, ) ) + lint_key: str = sql_rule_lint_key(config=lint_config) + file_keys: dict[str, str | None] = { + file_path.relative_to(project_dir).as_posix(): sql_file_rule_key( + lint_key=lint_key, + codes=list(codes), + relative_path=file_path.relative_to(project_dir).as_posix(), + contents=contents, + expanded=file_path in project.sql_expansions, + ) + for file_path, contents in project_files.items() + if file_path not in model_paths + } + file_hits: dict[str, tuple[Finding, ...]] = _cached_sql_findings( + bucket=bucket, + keys={path: key for path, key in file_keys.items() if key is not None}, + ) file_path: Path - contents: str - for file_path, contents in project_files.items(): + for file_path in project_files: if file_path in model_paths: continue relative_path: str = file_path.relative_to(project_dir).as_posix() - file_identity: str | None = _sql_file_rule_identity( - file_path=file_path, - relative_path=relative_path, - contents=contents, - codes=codes, - dialect=_lint_identity(lint_config), - sql_expansions=project.sql_expansions, - ) - cached_file: tuple[Finding, ...] | None = ( - None - if file_identity is None - else _cached_sql_findings(entry=bucket.get(relative_path), identity=file_identity) - ) + file_identity: str | None = file_keys[relative_path] + cached_file: tuple[Finding, ...] | None = file_hits.get(relative_path) if cached_file is None: selected_paths.add(file_path) if file_identity is not None: @@ -617,12 +561,9 @@ def _run_sql_rules( if path in by_path: by_path[path].append(finding) for path, identity in written_identities.items(): - bucket[path] = { - "identity": identity, - "findings": [_finding_cache_payload(item) for item in by_path[path]], - } - _write_sql_rule_cache(project_dir=project_dir, bucket=bucket) - return _SqlRulesEvaluation( + bucket[path] = (identity, encode_sql_rule_findings(findings=by_path[path])) + write_sql_rule_cache(path=_sql_rule_cache_path(project_dir), entries=bucket) + return SqlRulesEvaluation( findings=with_duplicate_literal_hints(findings=tuple(findings), project_dir=project_dir), cache_hits=len(identities) - len(misses), cache_misses=len(misses), @@ -693,21 +634,6 @@ def sql_lint_config( ) -def _lint_identity(config: LintConfig) -> str: - """Cache identity of everything besides SQL text that changes SQL Rule findings.""" - - return json.dumps( - [ - config.dialect, - config.max_literal_length, - config.max_ranking_order_by, - sorted(config.relation_keys.items()), - config.allow_suppressions, - ], - separators=(",", ":"), - ) - - def _sql_rule_codes(rules: tuple[Rule, ...]) -> tuple[str, ...]: return tuple(rule.code for rule in rules if rule.code.startswith("SQBRSQL")) @@ -726,132 +652,44 @@ def _sql_model_rule_identities( codes: tuple[str, ...] = _sql_rule_codes(rules) if not codes: return {} - return { - path: _sql_rule_identity(model=model, codes=codes, dialect=dialect) + models: list[tuple[str, str, str, str | None]] = [ + (path, model.authored_sql, model.query_sql, _contract_text(model)) for path, model in _sql_rule_models(project).items() - } - - -def _sql_rule_identity(*, model: CompiledModelView, codes: tuple[str, ...], dialect: str) -> str: - digest: Any = hashlib.blake2b(digest_size=32) - digest.update(_SQL_RULE_CACHE_VERSION.encode()) - digest.update(_RULES_BUILD_IDENTITY.encode()) - digest.update(dialect.encode()) - digest.update("\0".join(codes).encode()) - digest.update(model.authored_sql.encode()) - digest.update(model.query_sql.encode()) - if model.dynamic_column_contract is not None: - digest.update( - json.dumps( - asdict(model.dynamic_column_contract), - sort_keys=True, - separators=(",", ":"), - default=str, - ).encode() - ) - return digest.hexdigest() + ] + keys: dict[str, str] = sql_model_rule_keys(lint_key=dialect, codes=list(codes), models=models) + return {path: keys[path] for path, *_ in models} -def _sql_file_rule_identity( - *, - file_path: Path, - relative_path: str, - contents: str, - codes: tuple[str, ...], - dialect: str, - sql_expansions: dict[Path, CompiledSqlExpansion], -) -> str | None: - """Identify non-model SQL whose lint depends only on its text; expansions return None.""" +def _contract_text(model: CompiledModelView) -> str | None: + """Canonical text of the dynamic column contract that SQL rule findings depend on.""" - if ( - MACRO_TOKEN in contents - or TEMPLATE_INTERPOLATION_START in contents - or file_path in sql_expansions - ): + if model.dynamic_column_contract is None: return None - digest: Any = hashlib.blake2b(digest_size=32) - for value in ( - _SQL_RULE_CACHE_VERSION, - "file", - _RULES_BUILD_IDENTITY, - dialect, - "\0".join(codes), - relative_path, - contents, - ): - digest.update(value.encode()) - digest.update(b"\0") - return digest.hexdigest() + return json.dumps( + asdict(model.dynamic_column_contract), + sort_keys=True, + separators=(",", ":"), + default=str, + ) def _sql_rule_cache_path(project_dir: Path) -> Path: return rules_bulk_cache_path(project_dir=project_dir, file_name=SQL_RULES_CACHE_FILE) -def _read_sql_rule_cache(project_dir: Path) -> dict[str, dict[str, object]]: +def _read_sql_rule_cache(project_dir: Path) -> dict[str, tuple[str, str]]: """Return usable entries so incompatible builds cannot suppress early lint.""" - path: Path = _sql_rule_cache_path(project_dir) - try: - payload: object = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, json.JSONDecodeError): - return {} - if ( - not isinstance(payload, dict) - or payload.get("version") != _SQL_RULE_CACHE_VERSION - or payload.get("build_identity") != _RULES_BUILD_IDENTITY - ): - return {} - entries: object = payload.get("entries") - if not isinstance(entries, dict): - return {} - return {str(key): value for key, value in entries.items() if isinstance(value, dict)} + return load_sql_rule_cache(path=_sql_rule_cache_path(project_dir)) def _cached_sql_findings( - *, entry: dict[str, object] | None, identity: str -) -> tuple[Finding, ...] | None: - if entry is None or entry.get("identity") != identity: - return None - values: object = entry.get("findings") - if not isinstance(values, list): - return None - try: - return tuple(decode_rule_finding(value) for value in values) - except (KeyError, RulesError, TypeError, ValueError): - return None + *, bucket: dict[str, tuple[str, str]], keys: dict[str, str] +) -> dict[str, tuple[Finding, ...]]: + """Decoded stored findings of each path whose key still matches; undecodable ones miss.""" - -def _finding_cache_payload(finding: Finding) -> dict[str, object]: - return { - "unevaluated": finding.unevaluated, - "code": finding.code, - "path": finding.path.as_posix(), - "line": finding.line, - "column": finding.column, - "message": finding.message, - "remediation": finding.remediation, - "fixable": finding.fixable, - } - - -def _write_sql_rule_cache(*, project_dir: Path, bucket: dict[str, dict[str, object]]) -> None: - path: Path = _sql_rule_cache_path(project_dir) - path.parent.mkdir(parents=True, exist_ok=True) - temporary: Path = path.with_suffix(f".tmp-{os.getpid()}") - temporary.write_text( - json.dumps( - { - "version": _SQL_RULE_CACHE_VERSION, - "build_identity": _RULES_BUILD_IDENTITY, - "entries": bucket, - }, - sort_keys=True, - separators=(",", ":"), - ), - encoding="utf-8", - ) - temporary.replace(path) + hits: dict[str, list[Finding]] = lookup_sql_rule_cache(entries=bucket, keys=list(keys.items())) + return {path: tuple(findings) for path, findings in hits.items()} def _selected_model_paths( diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index b9f34c04ad..d985125218 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -11,7 +11,6 @@ CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v2" CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 CUSTOM_RULE_PROJECT_SUBJECT: str = "" -CUSTOM_RULES_CACHE_VERSION: str = "custom-rules-reads-v1" RULES_CACHE_DIRECTORY_NAME: str = "rules-cache" RULES_BULK_CACHE_DIRECTORY_NAME: str = "bulk" CUSTOM_RULES_CACHE_FILE: str = "custom-rules.json" @@ -80,10 +79,6 @@ AST_SELECT_KIND: str = "select" AST_CTE_KIND: str = "cte" CONTRACT_ENFORCED: str = "enforced" -CONTRACT_BOOLEAN_NAME_CODE: str = "SQBRCONTRACT102" -CONTRACT_TIMESTAMP_NAME_CODE: str = "SQBRCONTRACT103" -CONTRACT_DATE_NAME_CODE: str = "SQBRCONTRACT104" -EMPTY_INPUT_TEST_CODE: str = "SQBRTEST203" TYPE_PROOF_RULE_CODES: frozenset[str] = frozenset({"SQBRCONTRACT105"}) BOOLEAN_TYPE: str = "BOOLEAN" @@ -163,7 +158,3 @@ FACT_DECLARATIONS_PUBLIC_CONSTANTS: str = "declarations.public_constants" FACT_DECLARATIONS_ENUMS: str = "declarations.enums" FACT_DECLARATIONS_CONSTANTS: str = "declarations.constants" -RULES_CONFIG_SECTION: str = "rules" -ALLOW_EXCEPTIONS_KEY: str = "allow_exceptions" -RULE_EXCEPTIONS_KEY: str = "rule_exceptions" -RULE_IGNORES_KEY: str = "rule_ignores" diff --git a/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py b/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py index d2cb8c0d2e..c9be51978e 100644 --- a/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py +++ b/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py @@ -6,7 +6,7 @@ from threading import Event from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.rule_engine._helpers.run.rules import prepare_sql_rules as prepare +from sqlbuild.rule_engine._helpers.run.rule_threads import prepare_sql_rules as prepare from sqlbuild.rule_engine.models import PreparedSqlLint diff --git a/src/sqlbuild/rule_engine/main/load_config.py b/src/sqlbuild/rule_engine/main/load_config.py index c615b541ae..eb3a354929 100644 --- a/src/sqlbuild/rule_engine/main/load_config.py +++ b/src/sqlbuild/rule_engine/main/load_config.py @@ -2,11 +2,11 @@ from pathlib import Path -from sqlbuild.rule_engine._helpers.engine.config import load_rules_config as _load_rules_config +from sqlbuild._native import load_rules_config as _load_rules_config from sqlbuild.rule_engine.models import RulesConfig def load_rules_config(*, project_dir: Path) -> RulesConfig: """Load strict rules configuration from a SQLBuild project.""" - return _load_rules_config(project_dir) + return _load_rules_config(project_dir=project_dir.resolve()) diff --git a/src/sqlbuild/rule_engine/main/run_rules.py b/src/sqlbuild/rule_engine/main/run_rules.py index c5ecda7344..ea50bdb95a 100644 --- a/src/sqlbuild/rule_engine/main/run_rules.py +++ b/src/sqlbuild/rule_engine/main/run_rules.py @@ -5,7 +5,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.rule_engine._helpers.run.rules import evaluate_rules +from sqlbuild.rule_engine._helpers.run.rule_threads import evaluate_rules from sqlbuild.rule_engine.models import ( PreparedSqlLint, RulesConfig, diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 23af930558..610da26686 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -9,6 +9,7 @@ from typing import Any from sqlbuild.compiler.compile.models import DeclarationScopeBuild, SqlExpansionContext +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint.models import LintRunResult from sqlbuild.rule_engine.types import FactKey, RuleCheck, RuleOptionValue, RuleSubject @@ -339,6 +340,31 @@ class PreparedSqlLint: project_dir: Path | None = None +@dataclass(frozen=True) +class SqlRulesEvaluation: + """SQL rule findings of one run, their cache accounting and elapsed time.""" + + findings: tuple[Finding, ...] + cache_hits: int + cache_misses: int + sql_ms: int = 0 + + +@dataclass(frozen=True) +class RuleRunPlan: + """The catalogue, selected rules, project slice and settings one rules run evaluates.""" + + effective_config: RulesConfig + project_dir: Path + catalogue: tuple[Rule, ...] + custom_payloads: list[dict[str, object]] + selected: tuple[Rule, ...] + skipped_type_proof_rules: tuple[str, ...] + model_paths: frozenset[str] | None + project: CompiledProjectView + config: RulesConfig + + @dataclass(frozen=True) class RulesRunResult: """Unified findings and phase measurements for configured rules.""" @@ -440,9 +466,9 @@ class NativeRulesEvaluation: @dataclass(frozen=True) class CustomRulesOutcome: - """Custom-rule finding payloads in canonical order and their cache accounting.""" + """Custom-rule findings in canonical order and their cache accounting.""" - findings: tuple[dict[str, object], ...] + findings: tuple[Finding, ...] cache_hits: int cache_misses: int custom_ms: int diff --git a/src/sqlbuild/rule_engine/types.py b/src/sqlbuild/rule_engine/types.py index cc82fc5b28..b74529ea48 100644 --- a/src/sqlbuild/rule_engine/types.py +++ b/src/sqlbuild/rule_engine/types.py @@ -26,7 +26,9 @@ type RuleOptionValue = bool | int | str | tuple[str, ...] | tuple[int, ...] type FactKey = tuple[str, ...] -type FindingRow = tuple[bool, str, str, int, int, str, str] +type CustomEvaluationRow = tuple[ + str, str, tuple[tuple[FactKey, ...], str | None] | None, list[dict[str, object]] +] type CustomRulePlan = dict[str, frozenset[str] | None] type CustomHostPlan = dict[str, list[str] | None] diff --git a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py index 070b623da8..f55f2a1e62 100644 --- a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py @@ -7,7 +7,12 @@ import pytest -from sqlbuild._native import read_latest_fingerprints, write_fingerprint +from sqlbuild._native import ( + normalize_source_freshness_data_version, + read_latest_fingerprints, + source_freshness_data_version_hash, + write_fingerprint, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -25,12 +30,6 @@ from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.bigquery._test_types import ( BigQueryBuildFlowTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py index 16a61aaa99..13f1b813ed 100644 --- a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py @@ -6,7 +6,12 @@ import pytest -from sqlbuild._native import read_latest_fingerprints, write_fingerprint +from sqlbuild._native import ( + normalize_source_freshness_data_version, + read_latest_fingerprints, + source_freshness_data_version_hash, + write_fingerprint, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -24,12 +29,6 @@ from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.databricks._test_types import ( DatabricksBuildFlowTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py index 26c87ee52e..03070bec2b 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py @@ -7,6 +7,10 @@ import pytest +from sqlbuild._native import ( + normalize_source_freshness_data_version, + source_freshness_data_version_hash, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -22,12 +26,6 @@ from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeBuildFlowTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/diff/helpers.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/diff/helpers.py index e2081558d3..02c910a1fc 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/diff/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/diff/helpers.py @@ -4,6 +4,7 @@ from decimal import Decimal from typing import cast +from sqlbuild._native import render_diff_output from sqlbuild.adapter.contract.models import ( ColumnInfo, RowDiffColumnResult, @@ -13,7 +14,6 @@ RowDiffTolerance, SchemaDiffResult, ) -from sqlbuild.cli.commands._helpers.diff.output import render_diff_output from sqlbuild.executor.diff.models import DiffExecutionResult, ModelDiffResult from tests.integration.src.sqlbuild.cli.commands._helpers.diff._test_types import ( DiffOutputIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py index 1f6b7b4de5..91cb48d23d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py @@ -3,15 +3,17 @@ from __future__ import annotations import sys +from functools import partial from pathlib import Path import pytest from sqlbuild.cli.commands import models as command_models from sqlbuild.cli.commands._helpers.audit.invocation import resolve_audit_invocation -from sqlbuild.cli.commands._helpers.check.invocation import resolve_check_invocation -from sqlbuild.cli.commands._helpers.seed.invocation import resolve_seed_invocation -from sqlbuild.cli.commands._helpers.test.invocation import resolve_test_invocation +from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( + resolve_command_invocation, + resolve_reported_command_invocation, +) from tests.integration.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( CommandInvocationTestCase, ) @@ -35,7 +37,10 @@ ), CommandInvocationTestCase( description="check text output reports progress on stdout", - resolve=resolve_check_invocation, + resolve=partial( + resolve_reported_command_invocation, + invocation_type=command_models.CheckInvocation, + ), request_type=command_models.CheckCommandRequest, json_output=False, expected_invocation_type_name="CheckInvocation", @@ -44,7 +49,11 @@ ), CommandInvocationTestCase( description="test json output reports progress on stderr", - resolve=resolve_test_invocation, + resolve=partial( + resolve_reported_command_invocation, + invocation_type=command_models.TestInvocation, + require_connection=True, + ), request_type=command_models.TestCommandRequest, json_output=True, expected_invocation_type_name="TestInvocation", @@ -53,7 +62,10 @@ ), CommandInvocationTestCase( description="seed text output omits shared progress reporters", - resolve=resolve_seed_invocation, + resolve=partial( + resolve_command_invocation, + invocation_type=command_models.SeedInvocation, + ), request_type=command_models.SeedCommandRequest, json_output=False, expected_invocation_type_name="SeedInvocation", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 7aa8240085..d4cc7e7c3b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1080,3 +1080,21 @@ class BuildManifestTestCase: description: str model_count: int expected_test_type: str + + +@dataclass(frozen=True) +class SqlRuleCacheSequenceTestCase: + """Compiles after each `(project toml, model SQL)` step and the cache counts each reports.""" + + description: str + steps: tuple[tuple[str, str], ...] + expected_hits_and_misses: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class CustomRuleCacheSequenceTestCase: + """Compiles after each `(rule source, model SQL)` step and the cache counts each reports.""" + + description: str + steps: tuple[tuple[str, str], ...] + expected_hits_and_misses: tuple[tuple[int, int], ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index a806a80730..cddca28fcd 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -13,16 +13,19 @@ from _pytest.monkeypatch import MonkeyPatch from duckdb import DuckDBPyConnection -from sqlbuild._native import discover_project_inputs, format_load_execution_json +from sqlbuild._native import ( + discover_project_inputs, + format_load_execution_json, + run_load_command, + run_test_command, + select_load_entries, +) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands._helpers.load.selection import select_load_entries from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.main.execution._audit import run_audit from sqlbuild.cli.commands.main.execution._build import run_build -from sqlbuild.cli.commands.main.execution._load import run_load from sqlbuild.cli.commands.main.execution._scenario import run_scenario -from sqlbuild.cli.commands.main.execution._test import run_test from sqlbuild.cli.commands.main.project._plan import run_plan from sqlbuild.cli.commands.models import ( AuditCommandRequest, @@ -1229,7 +1232,7 @@ def test_given_no_managed_source_read_ambiguity_when_building_then_source_deferr "audit": lambda **kwargs: run_audit(AuditCommandRequest(**kwargs)), "build": lambda **kwargs: run_build(BuildCommandRequest(**kwargs)), "scenario": lambda **kwargs: run_scenario(ScenarioTestCommandRequest(**kwargs)), - "test": lambda **kwargs: run_test(TestCommandRequest(**kwargs)), + "test": lambda **kwargs: run_test_command(request=TestCommandRequest(**kwargs)), } exit_code: int = command_runners[test_case.command]( project_dir=tmp_path, @@ -2049,8 +2052,8 @@ def test_given_source_loader_when_running_load_then_writes_source_table( write_repo_files(tmp_path, test_case.project_files) json_output_path: Path = tmp_path / "target" / "load.json" - exit_code: int = run_load( - LoadCommandRequest( + exit_code: int = run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, select=test_case.select, @@ -3028,7 +3031,9 @@ def test_given_source_loader_write_strategy_when_running_load_twice_then_writes_ write_repo_files(tmp_path, test_case.project_files) for _ in range(test_case.run_count): - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -3403,7 +3408,9 @@ def test_given_loader_cursor_context_has_no_value_when_running_load_then_passes_ connection.close() for _ in range(test_case.run_count): - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -3498,8 +3505,8 @@ def test_given_reload_flag_when_running_load_then_passes_reload_context_to_loade ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load( - LoadCommandRequest( + exit_code: int = run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, reload=test_case.reload, @@ -3547,8 +3554,8 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load( - LoadCommandRequest( + exit_code: int = run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, cursor_overrides=test_case.cursor_overrides, @@ -3891,7 +3898,9 @@ def test_given_generator_loader_with_inferred_columns_when_running_load_then_wri ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -3956,7 +3965,9 @@ def test_given_generator_loader_yields_multiple_rows_when_running_load_then_writ ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -4335,8 +4346,8 @@ def test_given_loader_writes_many_rows_when_running_load_then_formats_human_row_ write_repo_files(tmp_path, test_case.project_files) json_output_path: Path = tmp_path / "target" / "load.json" - exit_code: int = run_load( - LoadCommandRequest( + exit_code: int = run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, json_output_path=json_output_path, @@ -4394,7 +4405,9 @@ def test_given_loader_returns_empty_rows_when_running_load_then_writes_empty_dec ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -4455,7 +4468,9 @@ def test_given_self_managed_loader_when_running_load_then_uses_loader_written_ta write_repo_files(tmp_path, test_case.project_files) for _ in range(test_case.run_count): - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) assert exit_code == 0 connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) @@ -4558,7 +4573,9 @@ def test_given_loader_returns_conflicting_types_when_running_load_then_fails_cle ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load(LoadCommandRequest(project_dir=tmp_path, no_color=True)) + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) captured: CaptureResult[str] = capsys.readouterr() assert exit_code == test_case.expected_exit_code @@ -4884,8 +4901,8 @@ def test_given_invalid_load_selectors_when_running_load_then_it_raises_clear_err write_repo_files(tmp_path, test_case.project_files) with pytest.raises(CliUserError) as exc_info: - run_load( - LoadCommandRequest( + run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, select=test_case.select, @@ -4956,8 +4973,8 @@ def test_given_no_selected_managed_sources_when_running_load_then_it_does_not_co ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_load( - LoadCommandRequest( + exit_code: int = run_load_command( + request=LoadCommandRequest( project_dir=tmp_path, no_color=True, select=test_case.select, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d9efef4b99..68aecdd422 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -15,6 +15,7 @@ import sqlbuild.compiler.compile._helpers.render.macros as macros_module import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module +import sqlbuild.rule_engine._helpers.run.rule_threads as rule_threads_module import sqlbuild.rule_engine._helpers.run.rules as rules_module from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro @@ -25,6 +26,7 @@ from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostSplitIntegrationTestCase, + CustomRuleCacheSequenceTestCase, DynamicPivotRulesIntegrationTestCase, EarlyLintInputsTestCase, ExplicitContractOutputRuleIntegrationTestCase, @@ -32,6 +34,7 @@ NumericRangeDecisionIntegrationTestCase, RulePassIntegrationTestCase, RulesIntegrationTestCase, + SqlRuleCacheSequenceTestCase, TypedContractRuleIntegrationTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( @@ -41,6 +44,31 @@ write_project_files, ) +_CACHE_RULES_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n' +_CACHE_THRESHOLD_TOML: str = _CACHE_RULES_TOML + "thresholds = { max_literal_length = 7 }\n" +_CACHE_POLICY_TOML: str = _CACHE_THRESHOLD_TOML + "allow_exceptions = false\n" +_CACHE_SELECTION_TOML: str = _CACHE_POLICY_TOML.replace( + '"SQBRSQL004"', '"SQBRSQL004", "SQBRSQL001"' +) +_CACHE_MODEL_SQL: str = 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n' +_CACHE_EDITED_SQL: str = _CACHE_MODEL_SQL.replace("LIMIT 1", "LIMIT 2") +_CUSTOM_CACHE_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["XSQBRCACHE001"]\n' +) +_CUSTOM_CACHE_RULE: str = """from sqlbuild.rules import Finding, Model, RuleContext, rule + +@rule( + code="XSQBRCACHE001", + message="Models must live in the final directory", + remediation="Move this model beneath models/final/.", +) +def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: + return [] if "final" in model.path.parts else [ctx.finding(subject=model)] +""" +_CUSTOM_CACHE_EDITED_RULE: str = _CUSTOM_CACHE_RULE.replace('"final" in', '"staging" in') +_CUSTOM_CACHE_MODEL_SQL: str = 'MODEL (description "Orders");\nSELECT 1 AS order_id\n' +_CUSTOM_CACHE_EDITED_SQL: str = _CUSTOM_CACHE_MODEL_SQL.replace("SELECT 1", "SELECT 2") + _EARLY_LINT_STOP_TIMEOUT_S: float = 30.0 _EARLY_LINT_FINDINGS: tuple[tuple[str, str, int | None, int | None], ...] = ( ("SQBRSQL005", "models/order_cents.sql", 2, 116), @@ -671,7 +699,6 @@ def test_given_sql_rule_cache_from_another_native_build_when_compiling_then_rule test_case: RulesIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', @@ -687,11 +714,10 @@ def test_given_sql_rule_cache_from_another_native_build_when_compiling_then_rule cold_exit: int = main(command) cold: dict[str, object] = json.loads(capsys.readouterr().out) - monkeypatch.setattr( - rules_module, - "_RULES_BUILD_IDENTITY", - f"{rules_module._RULES_BUILD_IDENTITY}-rebuilt", - ) + cache_path: Path = rules_module._sql_rule_cache_path(tmp_path) + stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) + stored["build_identity"] = f"{stored['build_identity']}-rebuilt" + cache_path.write_text(json.dumps(stored), encoding="utf-8") rebuilt_exit: int = main(command) rebuilt: dict[str, object] = json.loads(capsys.readouterr().out) warm_exit: int = main(command) @@ -976,14 +1002,15 @@ def test_given_previous_build_rule_cache_when_compiling_then_lint_overlaps_with_ test_case: EarlyLintInputsTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: write_project_files(project_dir=tmp_path, files=EARLY_LINT_PROJECT_FILES) - with monkeypatch.context() as previous_build: - previous_build.setattr(rules_module, "_RULES_BUILD_IDENTITY", "previous-build") - previous_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( - project_dir=tmp_path, capsys=capsys - ) + previous_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( + project_dir=tmp_path, capsys=capsys + ) + cache_path: Path = rules_module._sql_rule_cache_path(tmp_path) + stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) + stored["build_identity"] = "previous-build" + cache_path.write_text(json.dumps(stored), encoding="utf-8") upgraded_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( project_dir=tmp_path, capsys=capsys ) @@ -1020,7 +1047,7 @@ def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_ ) lint_started: Event = Event() stop_observed: list[bool] = [] - original_run_lint: Callable[..., LintRunResult] = rules_module.run_lint + original_run_lint: Callable[..., LintRunResult] = rule_threads_module.run_lint def stop_awaiting_run_lint(**kwargs: Any) -> LintRunResult: lint_started.set() @@ -1028,7 +1055,7 @@ def stop_awaiting_run_lint(**kwargs: Any) -> LintRunResult: stop_observed.append(stop is not None and stop.wait(timeout=_EARLY_LINT_STOP_TIMEOUT_S)) return original_run_lint(**kwargs) - monkeypatch.setattr(rules_module, "run_lint", stop_awaiting_run_lint) + monkeypatch.setattr(rule_threads_module, "run_lint", stop_awaiting_run_lint) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) @@ -2788,3 +2815,96 @@ def required_prefix(*, model: Model, ctx: RuleContext) -> list[Finding]: assert exit_code == test_case.expected_exit_code assert payload["diagnostics"] == [] + + +@pytest.mark.parametrize( + "test_case", + [ + SqlRuleCacheSequenceTestCase( + description="cold misses, warm hits and each SQL rule input edit misses", + steps=( + (_CACHE_RULES_TOML, _CACHE_MODEL_SQL), + (_CACHE_RULES_TOML, _CACHE_MODEL_SQL), + (_CACHE_RULES_TOML, _CACHE_EDITED_SQL), + (_CACHE_RULES_TOML, _CACHE_EDITED_SQL), + (_CACHE_THRESHOLD_TOML, _CACHE_EDITED_SQL), + (_CACHE_POLICY_TOML, _CACHE_EDITED_SQL), + (_CACHE_SELECTION_TOML, _CACHE_EDITED_SQL), + (_CACHE_SELECTION_TOML, _CACHE_EDITED_SQL), + ), + expected_hits_and_misses=( + (0, 1), + (1, 0), + (0, 1), + (1, 0), + (0, 1), + (0, 1), + (0, 1), + (1, 0), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_sql_rule_cache_when_inputs_change_then_only_unchanged_runs_hit( + test_case: SqlRuleCacheSequenceTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + observed: list[tuple[int, int]] = [] + for project_toml, model_sql in test_case.steps: + (tmp_path / "sqlbuild_project.toml").write_text(project_toml, encoding="utf-8") + model.write_text(model_sql, encoding="utf-8") + _ = main(["--project-dir", str(tmp_path), "compile", "--json"]) + timings: dict[str, int] = cast( + dict[str, int], json.loads(capsys.readouterr().out)["compile_timings"] + ) + observed.append((timings["rule_cache_hits"], timings["rule_cache_misses"])) + + assert tuple(observed) == test_case.expected_hits_and_misses + + +@pytest.mark.parametrize( + "test_case", + [ + CustomRuleCacheSequenceTestCase( + description=( + "the custom rule misses cold and after a source edit, and hits through a model " + "SQL edit it never read; the built-in model rules add one count per run" + ), + steps=( + (_CUSTOM_CACHE_RULE, _CUSTOM_CACHE_MODEL_SQL), + (_CUSTOM_CACHE_RULE, _CUSTOM_CACHE_MODEL_SQL), + (_CUSTOM_CACHE_RULE, _CUSTOM_CACHE_EDITED_SQL), + (_CUSTOM_CACHE_EDITED_RULE, _CUSTOM_CACHE_EDITED_SQL), + (_CUSTOM_CACHE_EDITED_RULE, _CUSTOM_CACHE_EDITED_SQL), + (_CUSTOM_CACHE_RULE, _CUSTOM_CACHE_EDITED_SQL), + ), + expected_hits_and_misses=((0, 2), (2, 0), (1, 1), (0, 2), (2, 0), (0, 2)), + ) + ], + ids=lambda case: case.description, +) +def test_given_custom_rule_cache_when_inputs_change_then_only_unchanged_runs_hit( + test_case: CustomRuleCacheSequenceTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(_CUSTOM_CACHE_TOML, encoding="utf-8") + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + rule_file: Path = tmp_path / "rules" / "layout.py" + rule_file.parent.mkdir() + observed: list[tuple[int, int]] = [] + for rule_source, model_sql in test_case.steps: + rule_file.write_text(rule_source, encoding="utf-8") + model.write_text(model_sql, encoding="utf-8") + _ = main(["--project-dir", str(tmp_path), "compile", "--json"]) + timings: dict[str, int] = cast( + dict[str, int], json.loads(capsys.readouterr().out)["compile_timings"] + ) + observed.append((timings["rule_cache_hits"], timings["rule_cache_misses"])) + + assert tuple(observed) == test_case.expected_hits_and_misses diff --git a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py index f0a8e3f9af..04d94ba4fc 100644 --- a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py +++ b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py @@ -9,10 +9,10 @@ import duckdb import pytest +from sqlbuild._native import execute_query_diff from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands._helpers.diff.execution import execute_query_diff from sqlbuild.cli.commands.models import DiffCommandRequest, QueryDiffPreparation from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index f34f336cba..4a8e449605 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -5,7 +5,12 @@ import pytest -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + build_dbt_manifest_index, + discover_project_inputs, +) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.graph.core import ( @@ -13,7 +18,6 @@ dbt_model_graph_key, expand_combined_downstream, ) -from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import ( DbtCliOptions, diff --git a/tests/unit/scripts/native_module_paths/__init__.py b/tests/unit/scripts/native_module_paths/__init__.py new file mode 100644 index 0000000000..e017e10480 --- /dev/null +++ b/tests/unit/scripts/native_module_paths/__init__.py @@ -0,0 +1 @@ +"""Native module path tooling tests.""" diff --git a/tests/unit/scripts/native_module_paths/_helpers/__init__.py b/tests/unit/scripts/native_module_paths/_helpers/__init__.py new file mode 100644 index 0000000000..d54a5ed796 --- /dev/null +++ b/tests/unit/scripts/native_module_paths/_helpers/__init__.py @@ -0,0 +1 @@ +"""Native module path helper tests.""" diff --git a/tests/unit/scripts/native_module_paths/_helpers/_test_types.py b/tests/unit/scripts/native_module_paths/_helpers/_test_types.py new file mode 100644 index 0000000000..db04ab1cdc --- /dev/null +++ b/tests/unit/scripts/native_module_paths/_helpers/_test_types.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeSourceModulePathTestCase: + description: str + source: str + expected_errors: tuple[str, ...] + + +@dataclass(frozen=True) +class RepositoryModulePathTestCase: + description: str + expected_errors: tuple[str, ...] + expected_logger_names: tuple[str, ...] diff --git a/tests/unit/scripts/native_module_paths/_helpers/helpers.py b/tests/unit/scripts/native_module_paths/_helpers/helpers.py new file mode 100644 index 0000000000..f8c3865cc3 --- /dev/null +++ b/tests/unit/scripts/native_module_paths/_helpers/helpers.py @@ -0,0 +1,9 @@ +from scripts.native_module_paths._helpers.crates import native_module_path_errors +from scripts.native_module_paths._helpers.scan import scan_native_source + +SOURCE_LABEL: str = "lib.rs" + + +def native_source_errors(source: str) -> tuple[str, ...]: + """Scan one synthetic Rust file and resolve what it imports.""" + return native_module_path_errors(scan=scan_native_source(source=source, label=SOURCE_LABEL)) diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py new file mode 100644 index 0000000000..36668c4da2 --- /dev/null +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -0,0 +1,136 @@ +from pathlib import Path + +import pytest + +from scripts.native_module_paths._helpers.crates import ( + native_module_path_errors, + scan_native_crates, +) +from scripts.native_module_paths.models import NativeModulePathScan +from tests.unit.scripts.native_module_paths._helpers._test_types import ( + NativeSourceModulePathTestCase, + RepositoryModulePathTestCase, +) +from tests.unit.scripts.native_module_paths._helpers.helpers import native_source_errors + +EXCEPTIONS_MODULE: str = "sqlbuild.cli.commands.exceptions" + + +@pytest.mark.parametrize( + "test_case", + [ + NativeSourceModulePathTestCase( + description="module constant and helper attribute resolve", + source=( + f'const ERRORS_MODULE: &str = "{EXCEPTIONS_MODULE}";\n' + 'let error = imported(py, ERRORS_MODULE, "CliUserError")?;\n' + ), + expected_errors=(), + ), + NativeSourceModulePathTestCase( + description="literal names a module attribute", + source=f'let error = py.import("sqlbuild")?;\nlet name = "{EXCEPTIONS_MODULE}.CliUserError";\n', + expected_errors=(), + ), + NativeSourceModulePathTestCase( + description="logger constant is not imported", + source='const LOGGER_NAME: &str = "sqlbuild.compile";\n', + expected_errors=(), + ), + NativeSourceModulePathTestCase( + description="relation names are formatted from lower-case values", + source=( + 'let relation = format!("{schema}.{table}");\n' + 'let qualified = format!("{}.{}", text_of(schema)?, text_of(&alias)?);\n' + 'let temporary = format!(".{CACHE_FILE_NAME}.{}.tmp", id);\n' + ), + expected_errors=(), + ), + NativeSourceModulePathTestCase( + description="unknown module literal", + source='const MISSING_MODULE: &str = "sqlbuild.cli.no_such_module";\n', + expected_errors=( + "lib.rs:1: 'sqlbuild.cli.no_such_module' is not an importable module or attribute.", + ), + ), + NativeSourceModulePathTestCase( + description="helper attribute missing from a module constant", + source=( + f'const ERRORS_MODULE: &str =\n "{EXCEPTIONS_MODULE}";\n' + 'let error = imported(\n py,\n ERRORS_MODULE,\n "NoSuchError",\n)?;\n' + ), + expected_errors=(f"lib.rs:3: '{EXCEPTIONS_MODULE}' has no attribute 'NoSuchError'.",), + ), + NativeSourceModulePathTestCase( + description="getattr missing from an imported literal", + source=f'let error = py\n .import("{EXCEPTIONS_MODULE}")?\n .getattr("NoSuchError")?;\n', + expected_errors=(f"lib.rs:2: '{EXCEPTIONS_MODULE}' has no attribute 'NoSuchError'.",), + ), + NativeSourceModulePathTestCase( + description="literal module and attribute pair missing the attribute", + source=f'const CALLS: [(&str, &str); 1] = [("{EXCEPTIONS_MODULE}", "NoSuchError")];\n', + expected_errors=(f"lib.rs:1: '{EXCEPTIONS_MODULE}' has no attribute 'NoSuchError'.",), + ), + NativeSourceModulePathTestCase( + description="package constant joined with a module name at runtime", + source='let module = format!("{RUNTIME_PACKAGE}.{module}");\n', + expected_errors=( + "lib.rs:1: module path is built at runtime; use a complete literal module path.", + ), + ), + NativeSourceModulePathTestCase( + description="module constant passed positionally to a joined template", + source='let module = format!(\n "{}.{}",\n EFFECTIVE_MODULE,\n module,\n);\n', + expected_errors=( + "lib.rs:1: module path is built at runtime; use a complete literal module path.", + ), + ), + NativeSourceModulePathTestCase( + description="package prefix completed at runtime", + source='let module = format!("sqlbuild.compiler.{module}");\n', + expected_errors=( + "lib.rs:1: 'sqlbuild.compiler.{module}' is not a complete dotted module path.", + "lib.rs:1: module path is built at runtime; use a complete literal module path.", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_native_source_when_checking_module_paths_then_reports_errors( + test_case: NativeSourceModulePathTestCase, +) -> None: + assert native_source_errors(test_case.source) == test_case.expected_errors + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryModulePathTestCase( + description="native crates import only existing Python modules and attributes", + expected_errors=(), + expected_logger_names=( + "sqlbuild.cli", + "sqlbuild.cli.commands._helpers.cost.collection", + "sqlbuild.cli.commands.classes.prepared_compile_artifacts", + "sqlbuild.compile", + "sqlbuild.fingerprints", + "sqlbuild.lineage", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_native_crates_when_checking_then_module_paths_resolve( + test_case: RepositoryModulePathTestCase, +) -> None: + repository: Path = Path(__file__).resolve().parents[5] + + scan: NativeModulePathScan = scan_native_crates(root=repository) + + assert scan.references + assert scan.logger_names == test_case.expected_logger_names + assert native_module_path_errors(scan=scan) == test_case.expected_errors + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/_test_types.py index 29bee8a035..cb04cad911 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/_test_types.py @@ -17,3 +17,10 @@ class CheckOutputProjectionCase: expected_first_duration: str expected_second_duration: str expected_generic_terminal_count: int + + +@dataclass(frozen=True) +class CheckMetadataRowsCase: + description: str + metadata: dict[str, object] + expected_row_fragment: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py index dc18c8a95e..53818cdf5a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py @@ -3,30 +3,16 @@ from __future__ import annotations from contextvars import Token +from datetime import UTC, datetime from io import StringIO -from types import SimpleNamespace -from typing import Any, cast import pytest -import sqlbuild.cli.commands._helpers.check.execution as check_execution -from sqlbuild._native import NativeProgressProjector -from sqlbuild.cli.commands._helpers.check.core import write_check_results -from sqlbuild.cli.commands._helpers.check.execution import execute_check_plan -from sqlbuild.cli.commands.models import CheckExecutionPreparation, CheckInvocation -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.executor.python_nodes.classes.run_state import PythonNodeRunState -from sqlbuild.executor.python_nodes.models import ( - PythonCheckExecutionResult, - PythonIngressLoaderExecutorResult, -) -from sqlbuild.observability import ( - EventDispatcher, - ResourceAttemptLifecycle, - dispatcher_scope, - invocation_scope, -) +from sqlbuild._native import NativeProgressProjector, write_check_results +from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult +from sqlbuild.python_nodes.types import PythonCheckSeverity from tests.unit.src.sqlbuild.cli.commands._helpers.check._test_types import ( + CheckMetadataRowsCase, CheckOutputProjectionCase, ) from tests.unit.src.sqlbuild.cli.commands._helpers.check.helpers import ( @@ -42,15 +28,6 @@ def write(self, text: str) -> int: return super().write(text) -class _CloseFailingAdapter: - def connect(self, connection_config: dict[str, object]) -> object: - return connection_config - - def close(self, connection: object) -> None: - del connection - raise OSError("controlled adapter close failure") - - @pytest.mark.parametrize( "test_case", ( @@ -122,90 +99,26 @@ def test_given_two_completed_checks_when_second_row_fails_then_close_renders_unc @pytest.mark.parametrize( "test_case", ( - CheckOutputProjectionCase( - description="adapter close failure leaves all executed check terminals unclaimed", - expected_first_duration="0.00s", - expected_second_duration="0.00s", - expected_generic_terminal_count=2, + CheckMetadataRowsCase( + description="metadata that JSON cannot encode does not block human rows", + metadata={"checked_at": datetime(2026, 1, 2, tzinfo=UTC)}, + expected_row_fragment="first", ), ), ids=lambda case: case.description, ) -def test_given_checks_complete_when_adapter_close_fails_then_close_renders_all_terminals( - monkeypatch: pytest.MonkeyPatch, - test_case: CheckOutputProjectionCase, +def test_given_non_json_metadata_when_rows_render_then_rows_are_written( + test_case: CheckMetadataRowsCase, ) -> None: stream: StringIO = StringIO() - projector: NativeProgressProjector = NativeProgressProjector(stream=stream, use_color=False) - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=projector.consume, accepts_opaque=False) - run_state: PythonNodeRunState = PythonNodeRunState() - ingress_result: PythonIngressLoaderExecutorResult = PythonIngressLoaderExecutorResult( - python_results=(), load_results=(), run_state=run_state - ) - monkeypatch.setattr(check_execution, "_execute_check_ingress", lambda **_: ingress_result) - monkeypatch.setattr(check_execution, "_execute_check_read_side", lambda **_: ()) - - def execute_checks(**_: Any) -> tuple[PythonCheckExecutionResult, ...]: - with ResourceAttemptLifecycle( - resource_id="check:first", - resource_kind="check", - resource_name="first", - run_id="check-run", - ): - pass - with ResourceAttemptLifecycle( - resource_id="check:second", - resource_kind="check", - resource_name="second", - run_id="check-run", - ): - pass - return check_results() - - monkeypatch.setattr(check_execution, "execute_python_checks", execute_checks) - invocation: CheckInvocation = cast( - CheckInvocation, - SimpleNamespace( - adapter=_CloseFailingAdapter(), - connection_config={}, - use_color=False, - ), + result: PythonCheckExecutionResult = PythonCheckExecutionResult( + node_name="first", + passed=False, + severity=PythonCheckSeverity.ERROR, + metadata=test_case.metadata, ) - preparation: CheckExecutionPreparation = cast( - CheckExecutionPreparation, - SimpleNamespace( - check_functions=(SimpleNamespace(name="first"), SimpleNamespace(name="second")), - python_graph=object(), - lifecycle_plan=object(), - relation_targets={}, - default_database=None, - default_schema=None, - project_relations=None, - ), - ) - pipeline_result: CompilePipelineResult = cast( - CompilePipelineResult, - SimpleNamespace( - project=SimpleNamespace( - run_id="check-run", effective_target_name=None, effective_vars={} - ), - plan_output=SimpleNamespace(source_map={}), - ), - ) - token: Token[NativeProgressProjector | None] = projector.install() - with invocation_scope("check-invocation"), dispatcher_scope(dispatcher): - with pytest.raises(OSError, match="controlled adapter close failure"): - execute_check_plan( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - providers=cast(Any, None), - ) - projector.restore(token) - projector.close() - output: str = stream.getvalue() - assert output.count("check first OK") == 1 - assert output.count("check second OK") == 1 - assert output.count(" OK ") == test_case.expected_generic_terminal_count + write_check_results(stream=stream, results=(result,), use_color=False) + + assert test_case.expected_row_fragment in stream.getvalue() + assert "FAIL" in stream.getvalue() diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index 8c449efd1f..f8245e03e2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -8,7 +8,6 @@ from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.cli.commands._helpers.check import core as check_core -from sqlbuild.cli.commands._helpers.check.core import check_dependency_closure from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation from sqlbuild.compiler.pipeline.models import CompilePipelineResult @@ -23,35 +22,6 @@ ) -@pytest.mark.parametrize( - "test_case", - ( - CheckPlanningTestCase( - description="transitive check dependency closure", - expected_names=frozenset({"asset", "task", "loader"}), - ), - ), - ids=lambda case: case.description, -) -def test_given_check_with_transitive_dependencies_when_planning_then_entire_closure_is_selected( - test_case: CheckPlanningTestCase, -) -> None: - graph: Mock = Mock( - upstream_deps={ - "quality_check": ("asset",), - "asset": ("task", "loader"), - "task": (), - "loader": (), - } - ) - - selected: frozenset[str] = check_dependency_closure( - graph=graph, check_names=frozenset({"quality_check"}) - ) - - assert selected == test_case.expected_names - - @pytest.mark.parametrize( "test_case", ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/__init__.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/__init__.py deleted file mode 100644 index 73b7c8bf6f..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Clone helper output tests.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py deleted file mode 100644 index 6457e8f576..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py +++ /dev/null @@ -1,57 +0,0 @@ -from dataclasses import dataclass - -from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult - - -@dataclass(frozen=True) -class RenderCloneOutputTestCase: - description: str - result: CloneExecutionResult - expected_fragments: tuple[str, ...] - expected_color_fragments: tuple[str, ...] = () - unexpected_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class RenderCloneItemLineTestCase: - description: str - index: int - total: int - item: CloneItemResult - expected_fragments: tuple[str, ...] - expected_line: str - relation_width: int | None = None - unexpected_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class CloneConnectionLifecycleTestCase: - description: str - expected_destination_config: dict[str, object] - - -@dataclass(frozen=True) -class RenderCloneFingerprintProgressTestCase: - description: str - completed: int - total: int - expected_line: str - - -@dataclass(frozen=True) -class RenderCloneFingerprintInterruptedTestCase: - description: str - completed: int - total: int - pending_identities: tuple[str, ...] - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class CloneFingerprintProgressReporterTestCase: - description: str - completed: int - total: int - pending_identities: tuple[str, ...] - expected_fragments: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_connections.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_connections.py deleted file mode 100644 index ed95052a97..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_connections.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import annotations - -from io import StringIO -from pathlib import Path -from typing import Any, cast - -import pytest - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.clone.connections import ( - close_clone_targets, - connect_clone_targets, -) -from sqlbuild.cli.commands.models import ( - CloneCommandRequest, - CloneConnectionContext, - CloneInvocation, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig -from tests.unit.src.sqlbuild.cli.commands._helpers.clone._test_types import ( - CloneConnectionLifecycleTestCase, -) - - -class _ConnectionTrackingAdapter: - def __init__(self) -> None: - self.connected_configs: list[dict[str, object]] = [] - self.closed_connections: list[object] = [] - self.connection: object = object() - - def connect(self, config: dict[str, Any]) -> object: - self.connected_configs.append(config) - return self.connection - - def close(self, connection: object) -> None: - self.closed_connections.append(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - CloneConnectionLifecycleTestCase( - description="connects and closes only the destination session", - expected_destination_config={"database": "destination.duckdb"}, - ) - ], - ids=lambda case: case.description, -) -def test_given_clone_targets_when_connecting_then_only_destination_session_is_opened_once( - test_case: CloneConnectionLifecycleTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - adapter: _ConnectionTrackingAdapter = _ConnectionTrackingAdapter() - monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.clone.connections.resolve_target_connection_config", - lambda **_: test_case.expected_destination_config, - ) - invocation: CloneInvocation = CloneInvocation( - effective_project_dir=Path("."), - discovered_inputs=DiscoveredProjectInputs( - project_config=ProjectConfig(name="test", adapter="duckdb"), - local_config=LocalConfig(), - ), - adapter_name="duckdb", - adapter=cast(BaseAdapter, adapter), - destination_target_name="dev", - use_color=False, - progress_stream=StringIO(), - ) - request: CloneCommandRequest = CloneCommandRequest( - project_dir=None, - no_color=True, - no_sql_validation=False, - origin_target_name="prod", - destination_target_name="dev", - hard_copy=False, - ) - - context: CloneConnectionContext = connect_clone_targets(request=request, invocation=invocation) - close_clone_targets(invocation=invocation, connection_context=context) - - assert adapter.connected_configs == [test_case.expected_destination_config] - assert adapter.closed_connections == [adapter.connection] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_fingerprint_progress.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_fingerprint_progress.py deleted file mode 100644 index fb959f4d0e..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_fingerprint_progress.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -from io import StringIO - -import pytest - -from sqlbuild.cli.commands._helpers.clone.fingerprint_progress import ( - create_clone_fingerprint_progress_reporter, -) -from sqlbuild.executor.clone.types import CloneFingerprintProgressReporter -from tests.unit.src.sqlbuild.cli.commands._helpers.clone._test_types import ( - CloneFingerprintProgressReporterTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - CloneFingerprintProgressReporterTestCase( - description="reports completed batch and unconfirmed identities after interruption", - completed=50, - total=52, - pending_identities=("model:orders_50", "model:orders_51"), - expected_fragments=( - "[50/52] propagated OK", - "[50/52] FAIL not confirmed: model:orders_50, model:orders_51", - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_partial_progress_when_interrupted_then_reporter_names_unconfirmed_fingerprints( - test_case: CloneFingerprintProgressReporterTestCase, -) -> None: - stream: StringIO = StringIO() - reporter: CloneFingerprintProgressReporter = create_clone_fingerprint_progress_reporter( - stream=stream, - use_color=False, - ) - - reporter( - completed=0, - total=test_case.total, - pending_identities=("model:orders_0", *test_case.pending_identities), - ) - reporter( - completed=test_case.completed, - total=test_case.total, - pending_identities=test_case.pending_identities, - ) - reporter.write_interrupted() - - output: str = stream.getvalue() - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_output.py deleted file mode 100644 index 896b49a3ac..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_output.py +++ /dev/null @@ -1,228 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.clone.output import ( - render_clone_fingerprint_interrupted_line, - render_clone_fingerprint_progress_line, - render_clone_item_line, - render_clone_output, -) -from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult -from sqlbuild.executor.clone.types import CloneAction, CloneStatus -from tests.unit.src.sqlbuild.cli.commands._helpers.clone._test_types import ( - RenderCloneFingerprintInterruptedTestCase, - RenderCloneFingerprintProgressTestCase, - RenderCloneItemLineTestCase, - RenderCloneOutputTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - RenderCloneFingerprintProgressTestCase( - description="confirmed batch reports cumulative progress", - completed=50, - total=386, - expected_line=" fingerprint[ 50/386] propagated OK", - ), - ), - ids=lambda case: case.description, -) -def test_given_confirmed_fingerprint_batch_when_rendering_then_progress_is_explicit( - test_case: RenderCloneFingerprintProgressTestCase, -) -> None: - rendered: str = render_clone_fingerprint_progress_line( - completed=test_case.completed, - total=test_case.total, - use_color=False, - ) - - assert rendered == test_case.expected_line - - -@pytest.mark.parametrize( - "test_case", - ( - RenderCloneFingerprintInterruptedTestCase( - description="interruption bounds pending identity context", - completed=50, - total=62, - pending_identities=tuple(f"model:orders_{index}" for index in range(12)), - expected_fragments=("[50/62] FAIL", "model:orders_0", "model:orders_9", "(+2 more)"), - unexpected_fragments=("orders_10",), - ), - ), - ids=lambda case: case.description, -) -def test_given_interrupted_fingerprint_batch_when_rendering_then_pending_context_is_bounded( - test_case: RenderCloneFingerprintInterruptedTestCase, -) -> None: - rendered: str = render_clone_fingerprint_interrupted_line( - completed=test_case.completed, - total=test_case.total, - pending_identities=test_case.pending_identities, - use_color=False, - ) - - assert all(fragment in rendered for fragment in test_case.expected_fragments) - assert all(fragment not in rendered for fragment in test_case.unexpected_fragments) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderCloneOutputTestCase( - description="summary footer reports counts and surfaces non-success messages", - result=CloneExecutionResult( - item_results=( - CloneItemResult( - name="fact_orders", - action=CloneAction.CLONED, - status=CloneStatus.SUCCESS, - ), - CloneItemResult( - name="stg_orders", - action=CloneAction.WARNING_MISSING_SOURCE, - status=CloneStatus.WARNING, - message="missing in origin environment", - ), - CloneItemResult( - name="dim_customers", - action=CloneAction.FAILED, - status=CloneStatus.FAILED, - message="boom", - ), - CloneItemResult( - name="add_tax", - action=CloneAction.RECREATED_FUNCTION, - status=CloneStatus.SUCCESS, - ), - ) - ), - expected_fragments=( - "stg_orders", - "missing in origin environment", - "dim_customers", - "boom", - "RECREATED_FUNCTIONS=1", - "\u2717 Completed with errors", - ), - expected_color_fragments=( - "\033[38;5;167m\u2717\033[0m \033[38;5;167m\033[1mCompleted with errors\033[0m", - "\033[2mCLONED=\033[0m\033[34m1\033[0m", - "\033[2mPASS=\033[0m\033[32m2\033[0m", - "\033[2mWARN=\033[0m\033[33m1\033[0m", - "\033[2mFAIL=\033[0m\033[38;5;167m1\033[0m", - ), - unexpected_fragments=("fact_orders", "add_tax"), - ), - ], - ids=lambda case: case.description, -) -def test_given_clone_result_when_rendering_summary_then_reports_footer_and_messages( - test_case: RenderCloneOutputTestCase, - capsys: pytest.CaptureFixture[str], -) -> None: - render_clone_output(result=test_case.result, use_color=False) - no_color_output: str = capsys.readouterr().out - - render_clone_output(result=test_case.result, use_color=True) - color_output: str = capsys.readouterr().out - - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in no_color_output - for fragment in test_case.unexpected_fragments: - assert fragment not in no_color_output - assert "\033[" not in no_color_output - for fragment in test_case.expected_color_fragments: - assert fragment in color_output - - -@pytest.mark.parametrize( - "test_case", - [ - RenderCloneItemLineTestCase( - description="streamed line shows position, action, origin->destination and status", - index=2, - total=5, - item=CloneItemResult( - name="fact_orders", - action=CloneAction.COPIED, - status=CloneStatus.SUCCESS, - origin_relation="PROD.MAIN.FACT_ORDERS", - destination_relation="DEV.MAIN.FACT_ORDERS", - duration_seconds=0.42, - ), - expected_fragments=( - "2/5", - "copied", - "prod.main.fact_orders", - "->", - "dev.main.fact_orders", - "OK", - "0.42s", - ), - relation_width=50, - expected_line=( - " 2/5 copied " - "prod.main.fact_orders -> dev.main.fact_orders OK 0.42s" - ), - unexpected_fragments=("PROD.MAIN.FACT_ORDERS", "DEV.MAIN.FACT_ORDERS"), - ), - RenderCloneItemLineTestCase( - description="mixed quoted relation names preserve quoted case", - index=5, - total=5, - item=CloneItemResult( - name="orders", - action=CloneAction.CLONED, - status=CloneStatus.SUCCESS, - origin_relation='PROD.MAIN."Orders"', - destination_relation='DEV.MAIN."Orders"', - ), - expected_fragments=('prod.main."Orders"', 'dev.main."Orders"'), - expected_line=( - ' 5/5 cloned prod.main."Orders" -> dev.main."Orders" OK' - ), - unexpected_fragments=("PROD.MAIN", "DEV.MAIN"), - ), - RenderCloneItemLineTestCase( - description="escaped bracket identifier preserves quoted case", - index=5, - total=5, - item=CloneItemResult( - name="orders", - action=CloneAction.CLONED, - status=CloneStatus.SUCCESS, - origin_relation="PROD.DBO.[My]]Table]", - destination_relation="DEV.DBO.[My]]Table]", - ), - expected_fragments=("prod.dbo.[My]]Table]", "dev.dbo.[My]]Table]"), - expected_line=( - " 5/5 cloned prod.dbo.[My]]Table] -> dev.dbo.[My]]Table] OK" - ), - unexpected_fragments=("[My]]table]",), - ), - ], - ids=lambda case: case.description, -) -def test_given_clone_item_when_rendering_line_then_shows_flow_and_status( - test_case: RenderCloneItemLineTestCase, -) -> None: - line: str = render_clone_item_line( - index=test_case.index, - total=test_case.total, - item=test_case.item, - use_color=False, - relation_width=test_case.relation_width, - ) - - assert line == test_case.expected_line - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in line - for fragment in test_case.unexpected_fragments: - assert fragment not in line diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py index 51d0a04e06..8df69e144c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py @@ -1,9 +1,6 @@ from dataclasses import dataclass -from pathlib import Path -from typing import Literal -from sqlbuild.cli.commands.models import DiffExampleRenderOptions -from sqlbuild.executor.diff.models import DiffExecutionResult, FullDiffSizeLimits +from sqlbuild.executor.diff.models import DiffExecutionResult @dataclass(frozen=True) @@ -17,54 +14,3 @@ class RenderDiffOutputTestCase: max_column_examples: int max_row_only_examples: int expected_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class RenderDiffEvidenceTestCase: - description: str - left_value: object - right_value: object - options: DiffExampleRenderOptions - expected_left_fragment: str - expected_right_fragment: str - expected_first_difference: int | None - expected_truncated: bool - expected_suppressed: bool - - -@dataclass(frozen=True) -class FullDiffSizeLimitsTestCase: - description: str - full: bool - schema_only: bool - bounded: str | None - prod_max_full_rows: int | Literal["unlimited"] | None - dev_max_full_rows: int | Literal["unlimited"] | None - expected_limits: FullDiffSizeLimits | None - - -@dataclass(frozen=True) -class DiffSizeGuardErrorTestCase: - description: str - select: tuple[str, ...] - unique_key_override: tuple[str, ...] - has_cursor: bool - expected_full_command: str - expected_bounded_command: str | None - expected_message_fragments: tuple[str, ...] - expected_absent_fragments: tuple[str, ...] = () - project_dir: Path | None = None - - -@dataclass(frozen=True) -class ParseDiffNameRangeTestCase: - description: str - name_range: str - expected_result: tuple[str, str | None] - - -@dataclass(frozen=True) -class ParseDiffNameRangeErrorTestCase: - description: str - name_range: str | None - expected_code: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/helpers.py deleted file mode 100644 index 5767145518..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/helpers.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Shared builders for diff CLI helper unit tests.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Literal - -from sqlbuild.cli.commands.models import DiffCommandRequest -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.diff.models import FullDiffModelSize, FullDiffSideSize -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - TargetConfig, - TargetDiffConfig, -) - - -def diff_request( - *, - full: bool = False, - schema_only: bool = False, - bounded: str | None = None, - select: tuple[str, ...] = ("orders",), - unique_key_override: tuple[str, ...] = (), - project_dir: Path | None = None, -) -> DiffCommandRequest: - """Return a model diff request from prod with the TO target left to the default.""" - - return DiffCommandRequest( - project_dir=project_dir, - no_color=True, - no_sql_validation=False, - from_name="prod", - to_name=None, - full=full, - schema_only=schema_only, - bounded=bounded, - select=select, - unique_key_override=unique_key_override, - ) - - -def prod_dev_inputs( - *, - prod_max_full_rows: int | Literal["unlimited"] | None, - dev_max_full_rows: int | Literal["unlimited"] | None, -) -> DiscoveredProjectInputs: - """Return discovered inputs with prod and dev targets carrying the given diff limits.""" - - return DiscoveredProjectInputs( - project_config=ProjectConfig( - name="shop", - adapter="duckdb", - targets={ - "prod": TargetConfig( - schema="prod", diff=TargetDiffConfig(max_full_rows=prod_max_full_rows) - ), - "dev": TargetConfig( - schema="dev", diff=TargetDiffConfig(max_full_rows=dev_max_full_rows) - ), - }, - ), - local_config=LocalConfig(), - ) - - -def blocked_orders(*, has_cursor: bool) -> tuple[FullDiffModelSize, ...]: - """Return one blocked model: prod over its limit and dev a view of unknown size.""" - - return ( - FullDiffModelSize( - name="orders", - left=FullDiffSideSize( - target="prod", relation="prod.orders", row_count=12_500_000, max_rows=10_000_000 - ), - right=FullDiffSideSize( - target="dev", - relation="dev.orders", - row_count=None, - max_rows=10_000_000, - detail="no row count (view)", - ), - has_cursor=has_cursor, - ), - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_evidence.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_evidence.py deleted file mode 100644 index c4de2b5e67..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_evidence.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Tests for bounded diff example evidence.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.diff.evidence import ( - display_example_value, - render_example_pair, -) -from sqlbuild.cli.commands.models import DiffExampleRenderOptions, RenderedDiffExampleValue -from tests.unit.src.sqlbuild.cli.commands._helpers.diff._test_types import ( - RenderDiffEvidenceTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderDiffEvidenceTestCase( - description="late text difference keeps nearby context", - left_value=("a" * 200) + "left-tail", - right_value=("a" * 200) + "right-tail", - options=DiffExampleRenderOptions(max_value_length=30), - expected_left_fragment="left-tail", - expected_right_fragment="right-tail", - expected_first_difference=200, - expected_truncated=True, - expected_suppressed=False, - ), - RenderDiffEvidenceTestCase( - description="suppression retains lengths without values", - left_value="left-private-value", - right_value="right-private-value", - options=DiffExampleRenderOptions(max_value_length=30, suppress_values=True), - expected_left_fragment="18 chars", - expected_right_fragment="19 chars", - expected_first_difference=0, - expected_truncated=False, - expected_suppressed=True, - ), - RenderDiffEvidenceTestCase( - description="explicit unlimited rendering preserves complete values", - left_value=("a" * 200) + "left-tail", - right_value=("a" * 200) + "right-tail", - options=DiffExampleRenderOptions(max_value_length=None), - expected_left_fragment=("a" * 200) + "left-tail", - expected_right_fragment=("a" * 200) + "right-tail", - expected_first_difference=200, - expected_truncated=False, - expected_suppressed=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_unequal_values_when_rendering_evidence_then_bounds_are_explicit( - test_case: RenderDiffEvidenceTestCase, -) -> None: - left: RenderedDiffExampleValue - right: RenderedDiffExampleValue - left, right = render_example_pair( - left_value=test_case.left_value, - right_value=test_case.right_value, - options=test_case.options, - ) - - assert test_case.expected_left_fragment in display_example_value(left) - assert test_case.expected_right_fragment in display_example_value(right) - assert left.first_difference == test_case.expected_first_difference - assert right.first_difference == test_case.expected_first_difference - assert left.truncated is test_case.expected_truncated - assert right.truncated is test_case.expected_truncated - assert left.suppressed is test_case.expected_suppressed - assert right.suppressed is test_case.expected_suppressed diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_output.py index f071a0b79f..bc344f1ca8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_output.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import render_diff_output from sqlbuild.adapter.contract.models import ( ColumnInfo, RowDiffColumnResult, @@ -13,7 +14,6 @@ RowDiffTolerance, SchemaDiffResult, ) -from sqlbuild.cli.commands._helpers.diff.output import render_diff_output from sqlbuild.executor.diff.models import DiffExecutionResult, ModelDiffResult from tests.unit.src.sqlbuild.cli.commands._helpers.diff._test_types import ( RenderDiffOutputTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_size_guard.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_size_guard.py deleted file mode 100644 index fb4604316d..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_size_guard.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Default full diff limits and size guard error rendering.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, cast - -import pytest - -from sqlbuild.cli.commands._helpers.diff.size_guard import ( - build_diff_size_guard_error, - resolve_full_diff_size_limits, -) -from sqlbuild.cli.commands.exceptions import DiffSizeGuardError -from sqlbuild.executor.diff.exceptions import FullDiffSizeGuardError -from sqlbuild.executor.diff.models import FullDiffSizeLimits -from tests.unit.src.sqlbuild.cli.commands._helpers.diff._test_types import ( - DiffSizeGuardErrorTestCase, - FullDiffSizeLimitsTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.diff.helpers import ( - blocked_orders, - diff_request, - prod_dev_inputs, -) - - -@pytest.mark.parametrize( - "test_case", - [ - FullDiffSizeLimitsTestCase( - description="unset limits use the built-in default", - full=False, - schema_only=False, - bounded=None, - prod_max_full_rows=None, - dev_max_full_rows=None, - expected_limits=FullDiffSizeLimits( - left_target="prod", - right_target="dev", - left_max_rows=10_000_000, - right_max_rows=10_000_000, - ), - ), - FullDiffSizeLimitsTestCase( - description="each side uses its own target limit", - full=False, - schema_only=False, - bounded=None, - prod_max_full_rows=500, - dev_max_full_rows="unlimited", - expected_limits=FullDiffSizeLimits( - left_target="prod", right_target="dev", left_max_rows=500, right_max_rows=None - ), - ), - FullDiffSizeLimitsTestCase( - description="explicit full bypasses the guard", - full=True, - schema_only=False, - bounded=None, - prod_max_full_rows=1, - dev_max_full_rows=1, - expected_limits=None, - ), - FullDiffSizeLimitsTestCase( - description="explicit schema only bypasses the guard", - full=False, - schema_only=True, - bounded=None, - prod_max_full_rows=1, - dev_max_full_rows=1, - expected_limits=None, - ), - FullDiffSizeLimitsTestCase( - description="explicit bounded bypasses the guard", - full=False, - schema_only=False, - bounded="7d", - prod_max_full_rows=1, - dev_max_full_rows=1, - expected_limits=None, - ), - ], - ids=lambda case: case.description, -) -def test_given_mode_flags_and_target_limits_when_resolving_then_guard_limits_match( - test_case: FullDiffSizeLimitsTestCase, -) -> None: - limits: FullDiffSizeLimits | None = resolve_full_diff_size_limits( - request=diff_request( - full=test_case.full, schema_only=test_case.schema_only, bounded=test_case.bounded - ), - discovered_inputs=prod_dev_inputs( - prod_max_full_rows=test_case.prod_max_full_rows, - dev_max_full_rows=test_case.dev_max_full_rows, - ), - from_target="prod", - to_target="dev", - ) - - assert limits == test_case.expected_limits - - -@pytest.mark.parametrize( - "test_case", - [ - DiffSizeGuardErrorTestCase( - description="model with cursor offers bounded command", - select=("orders",), - unique_key_override=(), - has_cursor=True, - expected_full_command="sqb diff prod:dev --full --select orders", - expected_bounded_command="sqb diff prod:dev --bounded --select orders", - expected_message_fragments=( - "orders: prod 12,500,000 rows, limit 10,000,000; " - "dev size unknown: no row count (view), limit 10,000,000", - "Run one of these instead:", - "sqb diff prod:dev --schema-only --select orders", - ), - ), - DiffSizeGuardErrorTestCase( - description="model without cursor omits bounded command and keeps key flags", - select=("tag:daily", "path:models/marts"), - unique_key_override=("order_id", "line number"), - has_cursor=False, - expected_full_command=( - "sqb diff prod:dev --full --select tag:daily path:models/marts " - "--key order_id 'line number'" - ), - expected_bounded_command=None, - expected_message_fragments=("; no cursor",), - expected_absent_fragments=("--bounded",), - ), - DiffSizeGuardErrorTestCase( - description="project dir is a global flag placed before diff", - select=("orders",), - unique_key_override=(), - has_cursor=False, - expected_full_command=( - "sqb --project-dir projects/shop diff prod:dev --full --select orders" - ), - expected_bounded_command=None, - expected_message_fragments=(), - project_dir=Path("projects/shop"), - ), - ], - ids=lambda case: case.description, -) -def test_given_blocked_models_when_building_guard_error_then_names_sizes_and_commands( - test_case: DiffSizeGuardErrorTestCase, -) -> None: - error: DiffSizeGuardError = build_diff_size_guard_error( - request=diff_request( - select=test_case.select, - unique_key_override=test_case.unique_key_override, - project_dir=test_case.project_dir, - ), - from_target="prod", - to_target="dev", - error=FullDiffSizeGuardError( - "full diff stopped", blocked=blocked_orders(has_cursor=test_case.has_cursor) - ), - ) - - assert error.code == "C270" - assert error.exit_code == 2 - assert error.status == "incomplete" - assert test_case.expected_full_command in error.message - fragment: str - for fragment in test_case.expected_message_fragments: - assert fragment in error.message - for fragment in test_case.expected_absent_fragments: - assert fragment not in error.message - assert fragment not in str(error.help) - commands: dict[str, Any] = cast(dict[str, Any], error.details["commands"]) - assert commands["full"] == test_case.expected_full_command - assert commands["bounded"] == test_case.expected_bounded_command - models: list[dict[str, Any]] = cast(list[dict[str, Any]], error.details["models"]) - assert models[0]["from"]["row_count"] == 12_500_000 - assert models[0]["to"]["row_count"] is None - assert models[0]["to"]["exceeds_limit"] is True - assert "targets.prod.diff.max_full_rows and targets.dev.diff.max_full_rows" in str(error.help) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_validation.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_validation.py deleted file mode 100644 index bc5290d355..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_validation.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Diff target range parsing.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.diff.validation import parse_diff_name_range -from sqlbuild.cli.commands.exceptions import CliUserError -from tests.unit.src.sqlbuild.cli.commands._helpers.diff._test_types import ( - ParseDiffNameRangeErrorTestCase, - ParseDiffNameRangeTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ParseDiffNameRangeTestCase( - description="explicit range keeps both targets", - name_range="prod:dev", - expected_result=("prod", "dev"), - ), - ParseDiffNameRangeTestCase( - description="bare target leaves TO for the active target", - name_range="prod", - expected_result=("prod", None), - ), - ], - ids=lambda case: case.description, -) -def test_given_target_range_when_parsing_then_returns_from_and_optional_to( - test_case: ParseDiffNameRangeTestCase, -) -> None: - assert parse_diff_name_range(test_case.name_range) == test_case.expected_result - - -@pytest.mark.parametrize( - "test_case", - [ - ParseDiffNameRangeErrorTestCase( - description="missing range", name_range=None, expected_code="C208" - ), - ParseDiffNameRangeErrorTestCase( - description="empty target", name_range="", expected_code="C209" - ), - ParseDiffNameRangeErrorTestCase( - description="empty TO", name_range="prod:", expected_code="C209" - ), - ParseDiffNameRangeErrorTestCase( - description="empty FROM", name_range=":dev", expected_code="C209" - ), - ParseDiffNameRangeErrorTestCase( - description="too many separators", name_range="a:b:c", expected_code="C209" - ), - ], - ids=lambda case: case.description, -) -def test_given_malformed_target_range_when_parsing_then_raises_coded_error( - test_case: ParseDiffNameRangeErrorTestCase, -) -> None: - with pytest.raises(CliUserError) as error_info: - parse_diff_name_range(test_case.name_range) - - assert error_info.value.code == test_case.expected_code - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py index 91dbce4d32..328765c511 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py @@ -2,14 +2,12 @@ from datetime import UTC, datetime +from sqlbuild._native import source_freshness_data_version_hash from sqlbuild.adapter.contract.models import ( QueryResult, TableFreshnessMetadata, TableFreshnessRequest, ) -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.spec.contracts.models import ( SourceEntry, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py index 748dc3ff04..b738f4483d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py @@ -6,8 +6,8 @@ import pytest -from sqlbuild.cli.commands._helpers.load.invocation import _effective_loader_defaults -from sqlbuild.cli.commands._helpers.load.selection import ( +from sqlbuild._native import ( + resolve_loader_defaults, select_load_entries, select_load_reference_entries, ) @@ -150,7 +150,7 @@ def test_given_connection_namespace_when_resolving_load_defaults_then_matches_co local_config=LocalConfig(), ) - defaults: tuple[str | None, str | None] = _effective_loader_defaults( + defaults: tuple[str | None, str | None] = resolve_loader_defaults( discovered_inputs=discovered_inputs, selected_target=None, target_config=None, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py index 9da1136f09..760b9d84af 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py @@ -2,7 +2,7 @@ from pathlib import Path -from sqlbuild.integrations.dbt._helpers.planning.plan import build_dbt_interop_plan +from sqlbuild._native import build_dbt_interop_plan from sqlbuild.integrations.dbt.models import DbtInteropPlan, DbtInteropSelectionResult from sqlbuild.integrations.dbt.types import DbtInteropCommand from tests.unit.src.sqlbuild.cli.commands.main.dbt._test_types import DbtAutoInitTestCase diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index eb7202473b..f2cd5b643f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -155,3 +155,6 @@ class MachineProgressRoutingCase: json_output: Path | None expected_stdout_fragment: str expected_stderr_fragment: str + command: str | None = None + dbt_args: tuple[str, ...] = () + expected_stdout_excludes: str = "Project compile FAIL" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index 34d5834104..131bee4ac8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -47,8 +47,8 @@ def build_json_recording_handler( ) -> Callable[..., int]: """Build a fake command handler that records json output flags and returns exit_code.""" - def record_json_handler(*args: object) -> int: - received_args.append(extract_json_output_fields(args)) + def record_json_handler(*args: object, **kwargs: object) -> int: + received_args.append(extract_json_output_fields((*args, *kwargs.values()))) return exit_code return record_json_handler diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py index 0d920969dc..30e64d5637 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py @@ -135,6 +135,34 @@ def test_given_no_public_sink_when_observability_runs_then_provider_import_is_sk expected_stdout_fragment="", expected_stderr_fragment="Project compile START", ), + MachineProgressRoutingCase( + description="plan json mode keeps stdout free of progress", + json=True, + json_output=None, + expected_stdout_fragment="", + expected_stderr_fragment="Project compile START", + command="plan", + expected_stdout_excludes="Project compile", + ), + MachineProgressRoutingCase( + description="dbt plan json flag routes progress to stderr", + json=False, + json_output=None, + expected_stdout_fragment="", + expected_stderr_fragment="Project compile START", + command="dbt", + dbt_args=("--json", "--select", "orders"), + expected_stdout_excludes="Project compile", + ), + MachineProgressRoutingCase( + description="dbt plan without json keeps human stdout progress", + json=False, + json_output=None, + expected_stdout_fragment="Project compile START", + expected_stderr_fragment="", + command="dbt", + dbt_args=("--select", "orders"), + ), ), ids=lambda case: case.description, ) @@ -146,6 +174,8 @@ def test_given_json_routing_options_when_operation_runs_then_human_stream_contra args: CliNamespace = CliNamespace() args.json = test_case.json args.json_output = test_case.json_output + args.command = test_case.command + args.dbt_args = list(test_case.dbt_args) with cli_observability_scope(args=args, project_dir=tmp_path): with OperationLifecycle(operation_kind="project", operation_name="project_compile"): @@ -154,6 +184,7 @@ def test_given_json_routing_options_when_operation_runs_then_human_stream_contra captured: CaptureResult[str] = capsys.readouterr() assert test_case.expected_stdout_fragment in captured.out assert test_case.expected_stderr_fragment in captured.err + assert test_case.expected_stdout_excludes not in captured.out @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 8b48016772..435f92a3c7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -9,13 +9,13 @@ import pytest -from sqlbuild._native import write_execution_header -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.commands._helpers.test.sql_progress import ( +from sqlbuild._native import ( build_test_expectation_rows, resolve_test_name_width, + write_execution_header, ) +from sqlbuild.adapter.contract.models import LifeCycleEvent +from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.cli.commands.classes import build_progress_callbacks from sqlbuild.cli.commands.classes.build_progress_callbacks import ( BuildProgressCallbacks, diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py index a9591a700e..84ae015d09 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py @@ -8,10 +8,9 @@ import pytest -from sqlbuild._native import format_audit_execution_json +from sqlbuild._native import format_audit_execution_json, format_parameterized_test_label from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.commands._helpers.test.sql_progress import format_parameterized_test_label from sqlbuild.compiler.auditing.models import ( MeasurementThresholdBound, MeasurementThresholds, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py index 1813408fa0..e7d28eb393 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -4,6 +4,9 @@ from dataclasses import dataclass +from sqlbuild.compiler.migrations.models import MigrationRelation +from sqlbuild.compiler.migrations.types import OldNameViewEventType + @dataclass(frozen=True) class NewestEventTestCase: @@ -60,3 +63,40 @@ class OldNameViewAbsentColumnsTestCase: absent_columns: tuple[str, ...] expected_statuses: tuple[str, ...] expected_archive_name: str | None + + +@dataclass(frozen=True) +class PinnedMoveEventIdTestCase: + description: str + run_id: str + target_name: str | None + origin: MigrationRelation + destination: MigrationRelation + expected_event_id: str + + +@dataclass(frozen=True) +class PinnedFreshnessHashTestCase: + description: str + source_name: str + strategy: str + value_kind: str + data_version: str + expected_hash: str + + +@dataclass(frozen=True) +class PinnedColumnEventIdTestCase: + description: str + relation: MigrationRelation + origin_column: str + destination_column: str + expected_event_id: str + + +@dataclass(frozen=True) +class PinnedOldNameEventIdTestCase: + description: str + event_type: OldNameViewEventType + migration_event_id: str + expected_event_id: str diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py index 8d6e7e1191..c796410eca 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py @@ -4,13 +4,9 @@ from datetime import UTC, datetime, timedelta -from sqlbuild.compiler.migrations.main.deterministic_column_event_id import ( +from sqlbuild._native import ( deterministic_column_migration_event_id, -) -from sqlbuild.compiler.migrations.main.deterministic_event_id import ( deterministic_migration_event_id, -) -from sqlbuild.compiler.migrations.main.deterministic_old_name_view_event_id import ( deterministic_old_name_view_event_id, ) from sqlbuild.compiler.migrations.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py new file mode 100644 index 0000000000..e873a56e4a --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py @@ -0,0 +1,137 @@ +"""Persisted state identities pinned to the values the Python implementation stored.""" + +import pytest + +from sqlbuild._native import ( + deterministic_column_migration_event_id, + deterministic_migration_event_id, + deterministic_old_name_view_event_id, + source_freshness_data_version_hash, +) +from sqlbuild.compiler.migrations.models import MigrationRelation +from sqlbuild.compiler.migrations.types import OldNameViewEventType +from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( + PinnedColumnEventIdTestCase, + PinnedFreshnessHashTestCase, + PinnedMoveEventIdTestCase, + PinnedOldNameEventIdTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PinnedMoveEventIdTestCase( + description="move with a database-qualified origin", + run_id="run-1", + target_name="prod", + origin=MigrationRelation(database="Analytics", schema="Staging", name="Orders"), + destination=MigrationRelation(database=None, schema="Marts", name="orders_v2"), + expected_event_id="d6b1671e12032fd2efd6b3cd386bb1a7c6ca0fb0b23b70ac7460b35b0cf3eec2", + ), + PinnedMoveEventIdTestCase( + description="move with non-ASCII names and no target", + run_id="run-\u00fc", + target_name=None, + origin=MigrationRelation(database=None, schema=None, name="Stra\u00dfe"), + destination=MigrationRelation(database="DB", schema="S", name="T"), + expected_event_id="b35d6666a637e3ceff072be25d1847d304d618808b5d2504d8d3a7a159394d89", + ), + ], + ids=lambda case: case.description, +) +def test_given_move_when_hashing_natively_then_matches_stored_python_event_id( + test_case: PinnedMoveEventIdTestCase, +) -> None: + event_id: str = deterministic_migration_event_id( + run_id=test_case.run_id, + target_name=test_case.target_name, + origin=test_case.origin, + destination=test_case.destination, + ) + + assert event_id == test_case.expected_event_id + + +@pytest.mark.parametrize( + "test_case", + [ + PinnedColumnEventIdTestCase( + description="column rename on a qualified relation", + relation=MigrationRelation(database="Analytics", schema="Marts", name="Orders"), + origin_column="Customer_ID", + destination_column="customer_key", + expected_event_id="3c7209754b53f98b38b7dbea44c6a9f019f47cfa79a847edceea535095da352c", + ), + ], + ids=lambda case: case.description, +) +def test_given_column_rename_when_hashing_natively_then_matches_stored_python_event_id( + test_case: PinnedColumnEventIdTestCase, +) -> None: + event_id: str = deterministic_column_migration_event_id( + run_id="run-1", + target_name="prod", + relation=test_case.relation, + origin_column=test_case.origin_column, + destination_column=test_case.destination_column, + ) + + assert event_id == test_case.expected_event_id + + +@pytest.mark.parametrize( + "test_case", + [ + PinnedOldNameEventIdTestCase( + description="old-name view creation", + event_type=OldNameViewEventType.VIEW_CREATED, + migration_event_id="abc123", + expected_event_id="505320604de24bfcf9ca65ba63d373978bb5a26dedf83ad5917d8c7ab70efa73", + ), + ], + ids=lambda case: case.description, +) +def test_given_old_name_step_when_hashing_natively_then_matches_stored_python_event_id( + test_case: PinnedOldNameEventIdTestCase, +) -> None: + event_id: str = deterministic_old_name_view_event_id( + event_type=test_case.event_type, migration_event_id=test_case.migration_event_id + ) + + assert event_id == test_case.expected_event_id + + +@pytest.mark.parametrize( + "test_case", + [ + PinnedFreshnessHashTestCase( + description="timestamp source freshness version", + source_name="orders", + strategy="column", + value_kind="timestamp", + data_version="2024-01-01T00:00:00+00:00", + expected_hash="795f41b1b4ffbca0b6bb0b5da1a4ab3ad7a280c28e27e5289bd3c1a2732c5756", + ), + PinnedFreshnessHashTestCase( + description="string source freshness version with escapes", + source_name="cli\u00ebnts", + strategy="sql", + value_kind="string", + data_version='v"1', + expected_hash="7f6e8f6a16da5d7653b75617515e4643961a227202c378b84cba4d18f39a7d12", + ), + ], + ids=lambda case: case.description, +) +def test_given_data_version_when_hashing_natively_then_matches_stored_python_hash( + test_case: PinnedFreshnessHashTestCase, +) -> None: + data_version_hash: str = source_freshness_data_version_hash( + source_name=test_case.source_name, + strategy=test_case.strategy, + value_kind=test_case.value_kind, + data_version=test_case.data_version, + ) + + assert data_version_hash == test_case.expected_hash diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 76115fd470..6fe28897c9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import build_dbt_manifest_index from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -19,7 +20,6 @@ resolve_udf_references, ) from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver import ( DbtCompileReferenceResolver, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 318f7f9605..77a9ebd7ed 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -31,7 +31,7 @@ from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope -from sqlbuild.compiler.planner.main.commands.sql_test import build_test_command_plan +from sqlbuild.compiler.planner.main.commands._sql_test import build_test_command_plan from sqlbuild.compiler.planner.models import ( ChainStep, NativeSqlTestPlan, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 4faf51483d..c9bd338579 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -19,10 +19,10 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope +from sqlbuild.compiler.planner.main.commands._seed import build_seed_command_plan +from sqlbuild.compiler.planner.main.commands._seed_state import read_selected_seed_fingerprints +from sqlbuild.compiler.planner.main.commands._sql_test import build_test_command_plan from sqlbuild.compiler.planner.main.commands.audit import build_audit_command_plan -from sqlbuild.compiler.planner.main.commands.seed import build_seed_command_plan -from sqlbuild.compiler.planner.main.commands.seed_state import read_selected_seed_fingerprints -from sqlbuild.compiler.planner.main.commands.sql_test import build_test_command_plan from sqlbuild.compiler.planner.models import ( PlannerRelationsContext, PlannerScope, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py b/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py index be6fd6545f..cadbc02b3b 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild.compiler.source_freshness._helpers.age_policy import ( - evaluate_source_freshness_age_policy, -) +from sqlbuild._native import evaluate_source_freshness_age_policy from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy from tests.unit.src.sqlbuild.compiler.source_freshness._helpers._test_types import ( SourceFreshnessAgePolicyDurationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py index f69c76ceed..1eb327b456 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py @@ -5,12 +5,10 @@ import pytest +from sqlbuild._native import normalize_source_freshness_data_version from sqlbuild.adapter.contract.models import TableFreshnessMetadata from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) from sqlbuild.compiler.source_freshness.main.observation import ( observe_configured_source_freshness, ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index b17c55f509..8ca26d561c 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -5,6 +5,7 @@ from types import MappingProxyType from typing import Any +from sqlbuild._native import source_freshness_data_version_hash from sqlbuild.adapter.contract.models import ( RelationLookup, TableFreshnessMetadata, @@ -17,9 +18,6 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.source_freshness._helpers.sql import build_read_latest_sql from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py index aea4a773a2..639d906e1b 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild.compiler.source_freshness.main.age_policy import ( - evaluate_source_freshness_age_policy, -) +from sqlbuild._native import evaluate_source_freshness_age_policy from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py index 8ac57c0346..6d56c58452 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py @@ -6,15 +6,13 @@ import pytest +from sqlbuild._native import source_freshness_data_version_hash from sqlbuild.adapter.contract.models import TableFreshnessMetadata, TableFreshnessRequest from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.main._planning import ( build_direct_source_freshness_planning_result, ) -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py index 3927e117ba..793d2035ba 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py @@ -4,13 +4,9 @@ import pytest -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( +from sqlbuild._native import ( normalize_source_freshness_data_version, -) -from sqlbuild.compiler.source_freshness.main.record_equivalence import ( + source_freshness_data_version_hash, source_freshness_records_equivalent, ) from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord diff --git a/tests/unit/src/sqlbuild/executor/load/test_run.py b/tests/unit/src/sqlbuild/executor/load/test_run.py index 8f75b2c567..ecab9d43a5 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_run.py +++ b/tests/unit/src/sqlbuild/executor/load/test_run.py @@ -8,8 +8,7 @@ import pytest -from sqlbuild._native import NativeProgressProjector -from sqlbuild.cli.commands.classes.load_progress_reporter import LoadProgressReporter +from sqlbuild._native import LoadProgressReporter, NativeProgressProjector from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import SkipMode diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py index b4b5e1776a..83fc645c18 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -8,12 +8,10 @@ import duckdb +from sqlbuild._native import deterministic_column_migration_event_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.main.deterministic_column_event_id import ( - deterministic_column_migration_event_id, -) from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py index 523b555642..321a6da001 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py @@ -11,9 +11,9 @@ from sqlbuild._native import ( ExecutionEventWriter, TerminalEventIndex, + format_check_json, terminal_event_index_scope, ) -from sqlbuild.cli.commands._helpers.check.core import format_check_json from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py index f5c72ebb29..dc0ec6ecd7 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.integrations.dbt._helpers.profile.normalize import normalize_dbt_profile_connection +from sqlbuild._native import normalize_dbt_profile_connection from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import ( NormalizedDbtProfileConnection, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index 4dc3665ed0..aef13370a8 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import build_dbt_interop_plan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -16,7 +17,6 @@ build_failed_sqlbuild_model_names, build_merged_dbt_execution_argv, ) -from sqlbuild.integrations.dbt._helpers.planning.plan import build_dbt_interop_plan from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.main.pipeline import execute as execute_module from sqlbuild.integrations.dbt.main.pipeline import plan as plan_module diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py b/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py index 936f1bdaff..7d37df9880 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py @@ -2,13 +2,13 @@ import pytest +from sqlbuild._native import build_dbt_manifest_index from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.graph.core import ( build_dbt_combined_graph, expand_combined_downstream, expand_combined_upstream, ) -from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt.models import DbtCombinedGraph, DbtManifestIndex from tests.unit.src.sqlbuild.integrations.dbt._test_types import DbtCombinedGraphTestCase from tests.unit.src.sqlbuild.integrations.dbt.helpers import ( diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py index e0bf8805b1..a0f9cafdca 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py @@ -2,11 +2,11 @@ import pytest -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.integrations.dbt._helpers.manifest.core import ( +from sqlbuild._native import ( build_dbt_manifest_index, resolve_dbt_manifest_model, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.integrations.dbt.models import ( DbtManifestIndex, DbtManifestModel, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py index 300370e5fa..f386c5a12c 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py @@ -2,11 +2,7 @@ import pytest -from sqlbuild.integrations.dbt._helpers.planning.plan import ( - build_dbt_interop_plan, - format_dbt_interop_plan, - format_dbt_interop_plan_json, -) +from sqlbuild._native import build_dbt_interop_plan, render_dbt_interop_plan from sqlbuild.integrations.dbt.models import DbtInteropPlan, DbtInteropSelectionResult, DbtLsNode from tests.unit.src.sqlbuild.integrations.dbt._test_types import DbtPlanTestCase @@ -86,8 +82,8 @@ def test_given_ordinary_interop_selection_when_rendering_then_reports_selected_w warnings=test_case.warnings, ) - human: str = format_dbt_interop_plan(plan=plan, use_color=False) - json_output: str = format_dbt_interop_plan_json(plan) + human: str = render_dbt_interop_plan(plan=plan, json_output=False, use_color=False) + json_output: str = render_dbt_interop_plan(plan=plan, json_output=True, use_color=False) assert (plan.dbt_skip_reason is not None) is test_case.expected_dbt_skipped assert (plan.sqlbuild_skip_reason is not None) is test_case.expected_sqlbuild_skipped diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py index 77e56f82b8..bf0deca744 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py @@ -4,10 +4,10 @@ import pytest +from sqlbuild._native import build_dbt_manifest_index from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_ls_argv from sqlbuild.integrations.dbt._helpers.graph.core import build_dbt_combined_graph -from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt._helpers.planning.orchestration import plan_dbt_interop_command from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py index 2add74343b..b564c604da 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py @@ -2,9 +2,9 @@ import pytest +from sqlbuild._native import build_dbt_manifest_index from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.graph.core import build_dbt_combined_graph -from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt._helpers.selection.core import resolve_dbt_interop_sqlbuild_selection from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError from sqlbuild.integrations.dbt.models import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 20fb1f390c..6a5bc8a547 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -1,6 +1,7 @@ """Test helpers for native rules engine boundaries.""" import json +from collections.abc import Callable from dataclasses import dataclass, field from operator import attrgetter from pathlib import Path @@ -9,6 +10,7 @@ import pytest from sqlbuild._native import ( + NativeRulesRequest, project_with, ) from sqlbuild.compiler.compile.models import ( @@ -22,7 +24,7 @@ from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.rule_engine._helpers.engine import native +from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.run import native_rows @@ -45,6 +47,9 @@ _MODEL_SQL: str = "WITH final AS (SELECT 1 AS id) SELECT id FROM final" +_BUILD_RULES_REQUEST: Callable[..., NativeRulesRequest] = native_rows._native.build_rules_request + + def project_with_scope(*, index: ScopeIndex) -> CompiledProjectView: """Return a minimal compiled project carrying canonical scope facts.""" @@ -66,13 +71,16 @@ def captured_native_request( def build_rules_request( project_rows: dict[str, Any], - models: list[tuple[object, ...]], - sql_tests: list[tuple[object, ...]], - sql_scenarios: list[tuple[object, ...]], - ) -> tuple[object, tuple[int, int, int]]: + models: tuple[object, list[str]], + sql_tests: list[object], + sql_scenarios: list[object], + ) -> object: + request: NativeRulesRequest = _BUILD_RULES_REQUEST( + project_rows, models, sql_tests, sql_scenarios + ) captured.update(cast(dict[str, Any], json.loads(project_rows["header_json"]))) - captured.update(models=models, sql_tests=sql_tests, sql_scenarios=sql_scenarios) - return object(), (len(models), len(sql_tests), len(sql_scenarios)) + captured.update(models=models, **json.loads(request.sql_facts_json())) + return request def evaluate_rules_request(request: object) -> NativeRulesEvaluation: return NativeRulesEvaluation( @@ -398,3 +406,46 @@ def _incremental_project(models: tuple[IncrementalModelSpec, ...]) -> CompiledPr return project_with( project=projects[0], models=tuple(project.models[0] for project in projects) ) + + +def sql_test_summary(fact: dict[str, Any]) -> tuple[object, ...]: + """The SQL test fact fields the native rules request carries, in a compact comparable form.""" + + return ( + fact["source_path"], + fact["ownership_root"], + fact["block_index"], + fact["name"], + fact["explicit_name"], + fact["mode"], + ( + fact["expected_model_names"], + fact["assertion_names"], + fact["assertion_target_model_names"], + fact["target_model_names"], + ), + [(resource["kind"], resource["name"]) for resource in fact["tested_resources"]], + ( + fact["authored_ctes"], + fact["expected_ctes"], + fact["assertion_ctes"], + fact["has_macro_mocks"], + fact["has_model_query_overrides"], + ), + fact["case_name"], + ) + + +def scenario_summary(fact: dict[str, Any]) -> tuple[object, ...]: + """The SQL scenario fact fields the native rules request carries, in field order.""" + + return ( + fact["source_path"], + fact["ownership_root"], + fact["name"], + fact["description"], + fact["expected_model_names"], + fact["assertion_names"], + fact["assertion_target_model_names"], + fact["target_model_names"], + ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py index 6906304ca6..08dae222f7 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py @@ -218,7 +218,7 @@ def test_given_untracked_rule_reading_hidden_file_when_file_changes_then_rule_re test_case.expected_rerun_hits, 2 - test_case.expected_rerun_hits, ) - assert tuple(finding["path"] for finding in after_edit.findings) == ( + assert tuple(finding.path.as_posix() for finding in after_edit.findings) == ( test_case.expected_finding_paths ) @@ -278,7 +278,7 @@ def run_host_then_edit(**kwargs: Any) -> object: 2 - test_case.expected_rerun_hits, ) assert rerun.findings == oracle.findings - assert tuple(finding["path"] for finding in rerun.findings) == ( + assert tuple(finding.path.as_posix() for finding in rerun.findings) == ( test_case.expected_finding_paths ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index 51c428b117..fab374960c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -22,6 +22,8 @@ captured_native_request, direct_sql_test, model_sql_test, + scenario_summary, + sql_test_summary, ) from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project @@ -86,7 +88,7 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar ) assert len(request["sql_tests"]) == test_case.expected_test_count - assert request["sql_tests"][:3] == [ + assert [sql_test_summary(fact) for fact in request["sql_tests"][:3]] == [ ( f"tests/unit/{name}.sql", "tests/unit", @@ -96,14 +98,14 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar mode, ([], [], [], []), [(mode, name)], - None, + ([], [], [], False, False), None, ) for index, (mode, name) in enumerate( (("macro", "normalize"), ("udf", "tax"), ("table_fn", "items")), start=1 ) ] - assert request["sql_tests"][3] == ( + assert sql_test_summary(request["sql_tests"][3]) == ( "tests/unit/test_orders__keeps_paid.sql", "tests/unit", 1, @@ -116,7 +118,7 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar None, ) assert len(request["sql_scenarios"]) == test_case.expected_scenario_count - assert request["sql_scenarios"] == [ + assert [scenario_summary(fact) for fact in request["sql_scenarios"]] == [ ( "tests/scenarios/orders.sql", "tests/scenarios", diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py index cb858b6cd6..141ef13ceb 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py @@ -3,7 +3,7 @@ import pytest from sqlbuild._native import NativeRulesRequest -from sqlbuild.rule_engine._helpers.engine import native +from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native from sqlbuild.rule_engine._helpers.run import native_rows from sqlbuild.rule_engine.models import NativeRulesEvaluation diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py index 51c7aff77b..ce522e41d0 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py @@ -5,7 +5,8 @@ import pytest from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine import native +from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native +from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import RulesCacheConfig, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule from tests.unit.src.sqlbuild.rule_engine._helpers.run._test_types import ( @@ -42,7 +43,7 @@ def test_given_config_value_the_encoder_rejects_when_building_natively_then_rule config_values={"priority": test_case.rejected_value}, ) - with pytest.raises(native.RulesError, match=test_case.expected_message): + with pytest.raises(RulesError, match=test_case.expected_message): native.evaluate_native( project=project, config=RulesConfig(cache=RulesCacheConfig(enabled=False)),