diff --git a/Cargo.lock b/Cargo.lock index af77bdac45..af826cf0c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1211,6 +1211,13 @@ dependencies = [ name = "sqlbuild-core" version = "0.156.1" +[[package]] +name = "sqlbuild-cursor" +version = "0.156.1" +dependencies = [ + "sqlbuild-core", +] + [[package]] name = "sqlbuild-discovery" version = "0.156.1" @@ -1258,6 +1265,7 @@ dependencies = [ "sqlbuild-cli", "sqlbuild-config", "sqlbuild-core", + "sqlbuild-cursor", "sqlbuild-discovery", "sqlbuild-model-config", "sqlbuild-pysource", diff --git a/Cargo.toml b/Cargo.toml index 2f981266b3..ad79fd0a8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "crates/sqlbuild-core", + "crates/sqlbuild-cursor", "crates/sqlbuild-pysource", "crates/sqlbuild-cache", "crates/sqlbuild-sqltext", @@ -24,6 +25,7 @@ resolver = "2" [workspace.metadata.native-layers] order = [ "sqlbuild-core", + "sqlbuild-cursor", "sqlbuild-pysource", "sqlbuild-cache", "sqlbuild-sqltext", @@ -93,6 +95,7 @@ sqlbuild-cache = { path = "crates/sqlbuild-cache" } sqlbuild-cli = { path = "crates/sqlbuild-cli" } sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } +sqlbuild-cursor = { path = "crates/sqlbuild-cursor" } sqlbuild-discovery = { path = "crates/sqlbuild-discovery" } sqlbuild-render = { path = "crates/sqlbuild-render" } sqlbuild-refactor = { path = "crates/sqlbuild-refactor" } diff --git a/crates/README.md b/crates/README.md index 38b90d60fb..f957e37ee2 100644 --- a/crates/README.md +++ b/crates/README.md @@ -7,10 +7,11 @@ that can be tested and benchmarked with `cargo test` alone. | Crate | Responsibility | |---|---| | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | +| `sqlbuild-cursor` | Microbatch cursor algebra: typed cursor values, grain flooring and stepping, half-open interval clamping, merging and batch splitting, timestamp text normalization and calendar durations, with Python `datetime` results and errors. Zones other than fixed offsets are resolved through a caller-supplied lookup. | | `sqlbuild-pysource` | Static reading of project Python source with the ruff parser: macro module structure, literal SQL arguments, definition locations and string-literal edits. It never imports or runs user code; CPython stays the authority on whether a file is valid Python. | | `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. | -| `sqlbuild-config` | Configuration file reading without Python: TOML, YAML 1.2 with the core schema and merge keys, and typed project and local config readers for the fields discovery needs. Errors carry their kind and position; YAML outside the supported subset is an `Unsupported` error. | +| `sqlbuild-config` | Configuration file reading without Python: TOML, YAML 1.2 with the core schema and merge keys, and typed project and local config readers for the fields discovery needs. It also writes the workspace files `sqb playground` and `sqb skills` create from packaged templates and skills. Errors carry their kind and position; YAML outside the supported subset is an `Unsupported` error. | | `sqlbuild-discovery` | Native project discovery and the only implementation of the model, SQL test, scenario, source and schema file collections and the declaration layout: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling and read errors, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises. | | `sqlbuild-scopes` | Declaration scopes, visibility, relationship grants, placement, report queries and the offline scope cache, including dialect-aware SQL-test and scenario scans. | | `sqlbuild-model-config` | MODEL configuration, columns and audits, template expansion over caller values, and dynamic-column name patterns using Rust regex syntax. | @@ -30,7 +31,7 @@ Dependencies point one way, from the top of this graph to the bottom: sqlbuild-python -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-config - -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-core + -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-cursor -> sqlbuild-core sqlbuild-cli -> sqlbuild-cache, sqlbuild-core ``` diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 824f507efa..2168789c56 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -16,6 +16,7 @@ pub mod explicit_references; pub mod fingerprint_storage; pub mod graph; pub mod lineage; +pub mod lineage_command; pub mod lineage_graph; pub mod manifest; pub mod python_plan; diff --git a/crates/sqlbuild-analysis/src/lineage_command/_helpers/column_text.rs b/crates/sqlbuild-analysis/src/lineage_command/_helpers/column_text.rs new file mode 100644 index 0000000000..d95ef04744 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/_helpers/column_text.rs @@ -0,0 +1,175 @@ +//! Human column lineage: dependency lists and column-trace trees. + +use std::collections::{HashMap, HashSet}; +use std::convert::Infallible; + +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::lineage_command::constants::{ + ALREADY_SHOWN, DOWNSTREAM_DIRECTION, HUMAN_COLUMN_TRACE_LIMIT, STAR_TRANSFORM, + STAR_TRANSFORM_LABEL, +}; +use crate::lineage_command::models::{ColumnTrace, LineageColumn, LineageColumnEdge}; +use crate::lineage_graph::main::trace_limit_note::trace_limit_note; +use crate::lineage_graph::main::tree_lines::tree_lines; +use crate::lineage_graph::models::TreeLine; +use crate::lineage_graph::types::TreeVisitor; + +/// `resource.column`. +fn column_id(column: &LineageColumn) -> String { + format!("{}.{}", column.resource_name, column.column_name) +} + +fn transform_label(edge: &LineageColumnEdge) -> &str { + if edge.transform == STAR_TRANSFORM { + STAR_TRANSFORM_LABEL + } else { + &edge.transform + } +} + +fn displayed(trace: &ColumnTrace) -> &[LineageColumnEdge] { + &trace.trace[..trace.trace.len().min(HUMAN_COLUMN_TRACE_LIMIT)] +} + +fn limit_note(trace: &ColumnTrace, style: CliStyle) -> Vec { + let notes: Vec = trace_limit_note( + i64::try_from(trace.trace.len()).unwrap_or(i64::MAX), + i64::try_from(HUMAN_COLUMN_TRACE_LIMIT).unwrap_or(i64::MAX), + ); + if notes.is_empty() { + return notes; + } + let mut lines: Vec = vec![String::new()]; + lines.extend(notes.iter().map(|note| style.muted(note))); + lines +} + +/// The trace as `source -> target transform` lines. +pub(crate) fn column_list(trace: &ColumnTrace, style: CliStyle) -> String { + if trace.trace.is_empty() { + return format!( + "Column dependencies\n\n{}", + style.section(&column_id(&trace.target)) + ); + } + let edges: &[LineageColumnEdge] = displayed(trace); + let width = |side: fn(&LineageColumnEdge) -> &LineageColumn| -> usize { + edges + .iter() + .map(|edge| column_id(side(edge)).chars().count()) + .max() + .unwrap_or(0) + }; + let left: usize = width(|edge| &edge.source); + let right: usize = width(|edge| &edge.target); + let mut lines: Vec = vec![style.title("Column dependencies"), String::new()]; + for edge in edges { + let source: String = column_id(&edge.source); + let target: String = column_id(&edge.target); + lines.push(format!( + "{}{} {} {}{} {}", + style.section(&source), + " ".repeat(left - source.chars().count()), + style.muted("->"), + style.section(&target), + " ".repeat(right - target.chars().count()), + style.muted(transform_label(edge)) + )); + } + lines.extend(limit_note(trace, style)); + lines.join("\n") +} + +/// The trace as a tree from the target column. +pub(crate) fn column_tree(trace: &ColumnTrace, style: CliStyle) -> String { + let target: String = column_id(&trace.target); + let mut lines: Vec = vec![ + format!( + "{} {} {}", + style.title("Column trace"), + style.section(&target), + style.muted(&trace.direction) + ), + String::new(), + ]; + if trace.trace.is_empty() { + lines.push(style.muted(" No column dependencies found")); + return lines.join("\n"); + } + let downstream: bool = trace.direction == DOWNSTREAM_DIRECTION; + let mut deps: HashMap> = HashMap::new(); + for edge in displayed(trace) { + let key: &LineageColumn = if downstream { + &edge.source + } else { + &edge.target + }; + deps.entry(column_id(key)).or_default().push(edge); + } + let mut tree = ColumnTree { + deps: &deps, + downstream, + style, + lines: Vec::new(), + }; + let roots: Vec<&LineageColumnEdge> = tree.sorted_edges(&target); + let Ok(()) = tree_lines(roots, HashSet::from([target]), &mut tree); + lines.extend(tree.lines); + lines.extend(limit_note(trace, style)); + lines.join("\n") +} + +/// Column-trace edges shown once per related column, sorted by that column. +struct ColumnTree<'a> { + deps: &'a HashMap>, + downstream: bool, + style: CliStyle, + lines: Vec, +} + +impl<'a> ColumnTree<'a> { + fn related_id(&self, edge: &LineageColumnEdge) -> String { + column_id(if self.downstream { + &edge.target + } else { + &edge.source + }) + } + + fn sorted_edges(&self, id: &str) -> Vec<&'a LineageColumnEdge> { + let mut edges: Vec<&'a LineageColumnEdge> = self.deps.get(id).cloned().unwrap_or_default(); + edges.sort_by_cached_key(|edge| self.related_id(edge)); + edges + } +} + +impl<'a> TreeVisitor<&'a LineageColumnEdge, String, Infallible> for ColumnTree<'a> { + fn key(&mut self, edge: &&'a LineageColumnEdge) -> Result { + Ok(self.related_id(edge)) + } + + fn line(&mut self, line: TreeLine<'_, &'a LineageColumnEdge>) -> Result<(), Infallible> { + let suffix: String = if line.repeated { + self.style.muted(ALREADY_SHOWN) + } else { + String::new() + }; + let edge: &LineageColumnEdge = line.node; + self.lines.push(format!( + "{}{}{} {}{suffix}", + line.prefix, + self.style.muted(line.branch), + self.style.section(&self.related_id(edge)), + self.style.muted(&format!("({})", transform_label(edge))) + )); + Ok(()) + } + + fn children( + &mut self, + edge: &&'a LineageColumnEdge, + ) -> Result, Infallible> { + Ok(self.sorted_edges(&self.related_id(edge))) + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/_helpers/graph.rs b/crates/sqlbuild-analysis/src/lineage_command/_helpers/graph.rs new file mode 100644 index 0000000000..7f5dd861fe --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/_helpers/graph.rs @@ -0,0 +1,102 @@ +//! Breadth-first closures and selected-graph assembly over a lineage index. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use crate::lineage_command::constants::{ + BOTH_DIRECTIONS, DOWNSTREAM_DIRECTION, UPSTREAM_DIRECTION, +}; +use crate::lineage_command::models::{LineageGraph, LineageIndex, LineageKey, LineageNode}; + +/// Edge lists by key. +pub(crate) type EdgeMap<'a> = HashMap<&'a LineageKey, &'a [LineageKey]>; + +/// `edges` as a lookup by key. +pub(crate) fn edge_map(edges: &[(LineageKey, Vec)]) -> EdgeMap<'_> { + edges + .iter() + .map(|(key, targets)| (key, targets.as_slice())) + .collect() +} + +/// Keys within `max_depth` edges of any start; a start is included only when reached again. +pub(crate) fn closure( + edges: &EdgeMap<'_>, + starts: &[LineageKey], + max_depth: Option, +) -> HashSet { + let mut visited: HashSet = HashSet::new(); + let mut frontier: VecDeque<(&LineageKey, i64)> = + starts.iter().map(|start| (start, 0)).collect(); + while let Some((current, depth)) = frontier.pop_front() { + if max_depth.is_some_and(|maximum| depth >= maximum) { + continue; + } + for neighbor in edges.get(current).copied().unwrap_or_default() { + if visited.insert(neighbor.clone()) { + frontier.push_back((neighbor, depth + 1)); + } + } + } + visited +} + +/// The keys `direction` reaches from `keys` within `depth`. +pub(crate) fn expand( + index: &LineageIndex, + keys: &[LineageKey], + (direction, depth): (&str, Option), +) -> HashSet { + let mut expanded: HashSet = HashSet::new(); + if direction == UPSTREAM_DIRECTION || direction == BOTH_DIRECTIONS { + expanded.extend(closure(&edge_map(&index.upstream), keys, depth)); + } + if direction == DOWNSTREAM_DIRECTION || direction == BOTH_DIRECTIONS { + expanded.extend(closure(&edge_map(&index.downstream), keys, depth)); + } + expanded +} + +/// `keys` sorted by resource type, then name. +pub(crate) fn sorted_keys<'a>(keys: impl IntoIterator) -> Vec { + let mut sorted: Vec = keys.into_iter().cloned().collect(); + sorted.sort(); + sorted.dedup(); + sorted +} + +/// Display nodes and the upstream edges between selected keys. +pub(crate) fn build_graph( + index: &LineageIndex, + selected: &HashSet, + (focus_keys, direction): (Vec, Option<&str>), +) -> LineageGraph { + let ordered: Vec = sorted_keys(selected); + let upstream: EdgeMap<'_> = edge_map(&index.upstream); + let mut edges: Vec<(LineageKey, LineageKey)> = Vec::new(); + for downstream in &ordered { + for upstream_key in upstream.get(downstream).copied().unwrap_or_default() { + if selected.contains(upstream_key) { + edges.push((upstream_key.clone(), downstream.clone())); + } + } + } + LineageGraph { + nodes: ordered + .iter() + .map(|key| { + index + .nodes + .get(key) + .cloned() + .unwrap_or_else(|| LineageNode { + key: key.clone(), + relative_path: None, + qualified_name: None, + }) + }) + .collect(), + edges, + focus_keys, + direction: direction.map(str::to_owned), + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/_helpers/graph_text.rs b/crates/sqlbuild-analysis/src/lineage_command/_helpers/graph_text.rs new file mode 100644 index 0000000000..32499947ce --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/_helpers/graph_text.rs @@ -0,0 +1,190 @@ +//! Human relation lineage: edge lists, dependency trees and graph summaries. + +use std::collections::{HashMap, HashSet}; +use std::convert::Infallible; + +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::lineage_command::constants::{ + ALREADY_SHOWN, BOTH_DIRECTIONS, DOWNSTREAM_DIRECTION, UPSTREAM_DIRECTION, +}; +use crate::lineage_command::models::{LineageGraph, LineageKey, LineageNode}; +use crate::lineage_graph::main::tree_lines::tree_lines; +use crate::lineage_graph::models::TreeLine; +use crate::lineage_graph::types::TreeVisitor; + +/// `type:name`. +pub(crate) fn node_id(key: &LineageKey) -> String { + format!("{}:{}", key.resource_type, key.name) +} + +fn format_key(key: &LineageKey, style: CliStyle) -> String { + format!( + "{}:{}", + style.muted(&key.resource_type), + style.section(&key.name) + ) +} + +fn format_node(node: &LineageNode, style: CliStyle) -> String { + let mut parts: Vec = vec![ + style.muted(&node.key.resource_type), + style.section(&node.key.name), + ]; + if let Some(path) = &node.relative_path { + parts.push(style.muted(path)); + } + parts.join(" ") +} + +/// The graph as `upstream -> downstream` lines, or its nodes when it has no edges. +pub(crate) fn graph_list(graph: &LineageGraph, style: CliStyle) -> String { + if graph.edges.is_empty() { + return graph + .nodes + .iter() + .map(|node| format_key(&node.key, style)) + .collect::>() + .join("\n"); + } + let width: usize = graph + .edges + .iter() + .map(|(upstream, _)| node_id(upstream).chars().count()) + .max() + .unwrap_or(0); + graph + .edges + .iter() + .map(|(upstream, downstream)| { + format!( + "{}{} {} {}", + format_key(upstream, style), + " ".repeat(width - node_id(upstream).chars().count()), + style.muted("->"), + format_key(downstream, style) + ) + }) + .collect::>() + .join("\n") +} + +/// A tree around the single focus node, or a summary of any other graph. +pub(crate) fn graph_tree(graph: &LineageGraph, style: CliStyle) -> String { + let (Some(direction), [focus]) = (graph.direction.as_deref(), graph.focus_keys.as_slice()) + else { + return graph_summary(graph, style); + }; + let nodes: HashMap<&LineageKey, &LineageNode> = + graph.nodes.iter().map(|node| (&node.key, node)).collect(); + let focus_text: String = nodes + .get(focus) + .map_or_else(String::new, |node| format_node(node, style)); + let mut lines: Vec = vec![format!( + "{} {focus_text} {}", + style.title("Lineage"), + style.muted(direction) + )]; + let sections: [(&str, bool); 2] = [(UPSTREAM_DIRECTION, true), (DOWNSTREAM_DIRECTION, false)]; + for (section, upstream) in sections { + if direction != section && direction != BOTH_DIRECTIONS { + continue; + } + let mut deps: HashMap<&LineageKey, Vec<&LineageKey>> = HashMap::new(); + for (parent, child) in &graph.edges { + let (from, to) = if upstream { + (child, parent) + } else { + (parent, child) + }; + deps.entry(from).or_default().push(to); + } + if direction == BOTH_DIRECTIONS { + lines.push(style.section(section)); + } + let mut tree = DependencyTree { + deps: &deps, + nodes: &nodes, + style, + lines: Vec::new(), + }; + let roots: Vec<&LineageKey> = tree.sorted_children(focus); + let Ok(()) = tree_lines(roots, HashSet::from([focus]), &mut tree); + lines.extend(tree.lines); + } + lines.join("\n") +} + +fn graph_summary(graph: &LineageGraph, style: CliStyle) -> String { + let mut lines: Vec = vec![format!( + "{} {}", + style.title("Lineage graph"), + style.muted(&format!( + "({} nodes, {} edges)", + graph.nodes.len(), + graph.edges.len() + )) + )]; + if graph.edges.is_empty() { + lines.extend( + graph + .nodes + .iter() + .map(|node| format!(" {}", format_node(node, style))), + ); + } else { + lines.extend(graph.edges.iter().map(|(parent, child)| { + format!( + "{}{} {} {}", + style.muted(" - "), + format_key(parent, style), + style.muted("->"), + format_key(child, style) + ) + })); + } + lines.join("\n") +} + +/// Dependency nodes shown once each, children sorted by type and name. +struct DependencyTree<'a> { + deps: &'a HashMap<&'a LineageKey, Vec<&'a LineageKey>>, + nodes: &'a HashMap<&'a LineageKey, &'a LineageNode>, + style: CliStyle, + lines: Vec, +} + +impl<'a> DependencyTree<'a> { + fn sorted_children(&self, key: &LineageKey) -> Vec<&'a LineageKey> { + let mut children: Vec<&'a LineageKey> = self.deps.get(key).cloned().unwrap_or_default(); + children.sort(); + children + } +} + +impl<'a> TreeVisitor<&'a LineageKey, &'a LineageKey, Infallible> for DependencyTree<'a> { + fn key(&mut self, node: &&'a LineageKey) -> Result<&'a LineageKey, Infallible> { + Ok(*node) + } + + fn line(&mut self, line: TreeLine<'_, &'a LineageKey>) -> Result<(), Infallible> { + let suffix: String = if line.repeated { + self.style.muted(ALREADY_SHOWN) + } else { + String::new() + }; + let node: String = self + .nodes + .get(*line.node) + .map_or_else(String::new, |node| format_node(node, self.style)); + self.lines.push(format!( + "{}{node}{suffix}", + self.style.muted(&format!("{}{}", line.prefix, line.branch)) + )); + Ok(()) + } + + fn children(&mut self, node: &&'a LineageKey) -> Result, Infallible> { + Ok(self.sorted_children(node)) + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/_helpers/json.rs b/crates/sqlbuild-analysis/src/lineage_command/_helpers/json.rs new file mode 100644 index 0000000000..b703f526ce --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/_helpers/json.rs @@ -0,0 +1,144 @@ +//! JSON payloads of relation graphs, column traces and semantic uses. + +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::lineage_command::_helpers::graph_text::node_id; +use crate::lineage_command::models::{ + ColumnTrace, LineageColumn, LineageColumnEdge, LineageGraph, LineageNode, SemanticUse, +}; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn object(entries: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + entries + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ) +} + +fn integer(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, |value| { + JsonValue::Integer(JsonInteger::from(value)) + }) +} + +fn node(node: &LineageNode) -> JsonValue { + let mut entries: Vec<(&str, JsonValue)> = vec![ + ("id", text(&node_id(&node.key))), + ("name", text(&node.key.name)), + ("resource_type", text(&node.key.resource_type)), + ]; + if let Some(path) = &node.relative_path { + entries.push(("relative_path", text(path))); + } + if let Some(name) = &node.qualified_name { + entries.push(("qualified_name", text(name))); + } + object(entries) +} + +/// The selected graph's nodes, edges, focus and direction. +pub(crate) fn graph_json(graph: &LineageGraph) -> JsonValue { + let mut entries: Vec<(&str, JsonValue)> = vec![ + ( + "nodes", + JsonValue::Array(graph.nodes.iter().map(node).collect()), + ), + ( + "edges", + JsonValue::Array( + graph + .edges + .iter() + .map(|(upstream, downstream)| { + object(vec![ + ("from", text(&node_id(upstream))), + ("to", text(&node_id(downstream))), + ]) + }) + .collect(), + ), + ), + ]; + if !graph.focus_keys.is_empty() { + entries.push(( + "focus", + JsonValue::Array( + graph + .focus_keys + .iter() + .map(|key| text(&node_id(key))) + .collect(), + ), + )); + } + if let Some(direction) = &graph.direction { + entries.push(("direction", text(direction))); + } + object(entries) +} + +fn column(column: &LineageColumn) -> JsonValue { + object(vec![ + ("resource_type", text(&column.resource_type)), + ("resource_name", text(&column.resource_name)), + ("column_name", text(&column.column_name)), + ]) +} + +fn edge(edge: &LineageColumnEdge) -> JsonValue { + object(vec![ + ("source", column(&edge.source)), + ("target", column(&edge.target)), + ("transform", text(&edge.transform)), + ("confidence", text(&edge.confidence)), + ]) +} + +/// The traced column, its trace and the analysis metadata. +pub(crate) fn trace_json(trace: &ColumnTrace) -> JsonValue { + object(vec![ + ("target", column(&trace.target)), + ("direction", text(&trace.direction)), + ( + "metadata", + object(vec![ + ("mode", text(&trace.mode)), + ("max_depth", integer(trace.max_depth)), + ( + "analyzed_models", + integer(i64::try_from(trace.analyzed_model_count).ok()), + ), + ("truncated", JsonValue::Bool(trace.truncated)), + ]), + ), + ( + "trace", + JsonValue::Array(trace.trace.iter().map(edge).collect()), + ), + ]) +} + +/// One semantic use's payload. +pub(crate) fn use_json(semantic_use: &SemanticUse) -> JsonValue { + object(vec![ + ("consumer_model", text(&semantic_use.consumer_model)), + ("context", text(&semantic_use.context)), + ("expression", text(&semantic_use.expression)), + ( + "source", + object(vec![ + ("resource_type", text(&semantic_use.source.resource_type)), + ("resource_name", text(&semantic_use.source.resource_name)), + ("column", text(&semantic_use.source.column_name)), + ]), + ), + ("confidence", text(&semantic_use.confidence)), + ("line", integer(semantic_use.line)), + ("column", integer(semantic_use.column)), + ]) +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/_helpers/mod.rs b/crates/sqlbuild-analysis/src/lineage_command/_helpers/mod.rs new file mode 100644 index 0000000000..1d5bb5c47d --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod column_text; +pub(crate) mod graph; +pub(crate) mod graph_text; +pub(crate) mod json; +pub(crate) mod targets; diff --git a/crates/sqlbuild-analysis/src/lineage_command/_helpers/targets.rs b/crates/sqlbuild-analysis/src/lineage_command/_helpers/targets.rs new file mode 100644 index 0000000000..2ff661f282 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/_helpers/targets.rs @@ -0,0 +1,16 @@ +//! The error for a target that names no project resource. + +use crate::lineage_command::constants::GRAPH_OPERATORS; +use crate::lineage_command::errors::LineageError; + +/// `unknown lineage target`, pointing at `--direction` when the target uses graph operators. +#[must_use] +pub(crate) fn unknown_lineage_target(target: &str) -> LineageError { + let help: &str = if target.contains(GRAPH_OPERATORS) { + "graph operators are not supported in lineage targets; \ + use --direction upstream|downstream|both and --depth N" + } else { + "use a model, source or seed name, optionally prefixed like model:orders" + }; + LineageError::new("C305", &format!("unknown lineage target '{target}'")).with_help(help) +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/constants.rs b/crates/sqlbuild-analysis/src/lineage_command/constants.rs new file mode 100644 index 0000000000..58b37c830c --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/constants.rs @@ -0,0 +1,23 @@ +//! Directions, formats and labels of `sqb lineage`. + +pub const UPSTREAM_DIRECTION: &str = "upstream"; +pub const DOWNSTREAM_DIRECTION: &str = "downstream"; +pub const BOTH_DIRECTIONS: &str = "both"; +pub const JSON_OUTPUT_FORMAT: &str = "json"; +pub const LIST_OUTPUT_FORMAT: &str = "list"; +pub const UNLIMITED_DEPTH_VALUE: &str = "all"; +pub const COLUMN_TARGET_SEPARATOR: char = '.'; +pub const COLUMN_TARGET_KIND: &str = "column"; +pub const MODEL_RESOURCE_TYPE: &str = "model"; +pub const SELECTOR_INTERSECTION_MARKER: char = ','; +/// Column-lineage model count from which rich analysis shows a status spinner. +pub const RICH_STATUS_MODEL_THRESHOLD: usize = 100; +pub(crate) const KIND_SEPARATOR: char = ':'; +pub(crate) const GRAPH_OPERATORS: [char; 3] = ['+', '@', '*']; +/// Kind prefixes a lineage target may carry. +pub(crate) const TARGET_KINDS: [&str; 6] = ["model", "source", "seed", "udf", "table_fn", "column"]; +pub(crate) const HUMAN_COLUMN_TRACE_LIMIT: usize = 25; +pub(crate) const STAR_TRANSFORM: &str = "star"; +pub(crate) const STAR_TRANSFORM_LABEL: &str = "from SELECT *"; +pub(crate) const ALREADY_SHOWN: &str = " (already shown)"; +pub(crate) const DEPTH_MESSAGE: &str = "--depth must be a non-negative integer or 'all'"; diff --git a/crates/sqlbuild-analysis/src/lineage_command/errors.rs b/crates/sqlbuild-analysis/src/lineage_command/errors.rs new file mode 100644 index 0000000000..3c5ed85fc3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/errors.rs @@ -0,0 +1,30 @@ +//! User-facing `sqb lineage` failures. + +/// A CLI user error: its code, message and optional help. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LineageError { + pub code: &'static str, + pub message: String, + pub help: Option, +} + +impl LineageError { + /// An error without help text. + #[must_use] + pub fn new(code: &'static str, message: &str) -> Self { + Self { + code, + message: message.to_owned(), + help: None, + } + } + + /// The same error with help text. + #[must_use] + pub fn with_help(self, help: &str) -> Self { + Self { + help: Some(help.to_owned()), + ..self + } + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/column_candidates.rs b/crates/sqlbuild-analysis/src/lineage_command/main/column_candidates.rs new file mode 100644 index 0000000000..738c746784 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/column_candidates.rs @@ -0,0 +1,40 @@ +//! The models a column trace must analyse. + +use std::collections::HashSet; + +use crate::lineage_command::_helpers::graph::{EdgeMap, closure, edge_map}; +use crate::lineage_command::constants::{DOWNSTREAM_DIRECTION, MODEL_RESOURCE_TYPE}; +use crate::lineage_command::models::{ColumnCandidates, LineageIndex, LineageKey}; + +/// Models within `depth` of `key` in `direction`, and whether one more step reaches further. +#[must_use] +pub fn column_candidates( + index: &LineageIndex, + key: &LineageKey, + (direction, depth): (&str, Option), +) -> ColumnCandidates { + let edges: EdgeMap<'_> = edge_map(if direction == DOWNSTREAM_DIRECTION { + &index.downstream + } else { + &index.upstream + }); + let starts: &[LineageKey] = std::slice::from_ref(key); + let mut selected: HashSet = closure(&edges, starts, depth); + selected.insert(key.clone()); + let mut model_names: Vec = selected + .iter() + .filter(|selected| selected.resource_type == MODEL_RESOURCE_TYPE) + .map(|selected| selected.name.clone()) + .collect(); + model_names.sort(); + model_names.dedup(); + let truncated: bool = depth.is_some_and(|depth| { + let mut extended: HashSet = closure(&edges, starts, Some(depth + 1)); + extended.insert(key.clone()); + extended != selected + }); + ColumnCandidates { + model_names, + truncated, + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/lineage_path_nodes.rs b/crates/sqlbuild-analysis/src/lineage_command/main/lineage_path_nodes.rs new file mode 100644 index 0000000000..6ce7bb0db0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/lineage_path_nodes.rs @@ -0,0 +1,40 @@ +//! Resources on directed paths between two endpoints. + +use std::collections::HashSet; + +use crate::lineage_command::_helpers::graph::{EdgeMap, closure, edge_map}; +use crate::lineage_command::models::{LineageIndex, LineageKey}; + +/// Keys on downstream paths from `start` to `end`, or none when `end` is not downstream. +#[must_use] +pub fn lineage_path_nodes( + index: &LineageIndex, + start: &LineageKey, + end: &LineageKey, +) -> Vec { + let downstream: EdgeMap<'_> = edge_map(&index.downstream); + let reachable: HashSet = closure(&downstream, std::slice::from_ref(start), None); + if !reachable.contains(end) { + return Vec::new(); + } + let mut upstream_of_end: HashSet<&LineageKey> = HashSet::new(); + let mut stack: Vec<&LineageKey> = vec![end]; + while let Some(current) = stack.pop() { + if !upstream_of_end.insert(current) { + continue; + } + for (parent, children) in &index.downstream { + if children.contains(current) && (reachable.contains(parent) || parent == start) { + stack.push(parent); + } + } + } + let mut nodes: Vec = reachable + .iter() + .filter(|key| upstream_of_end.contains(key)) + .cloned() + .collect(); + nodes.push(start.clone()); + nodes.push(end.clone()); + nodes +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/lineage_stdout.rs b/crates/sqlbuild-analysis/src/lineage_command/main/lineage_stdout.rs new file mode 100644 index 0000000000..a632aaaa7c --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/lineage_stdout.rs @@ -0,0 +1,57 @@ +//! The text `sqb lineage` writes to stdout. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::lineage_command::_helpers::json::use_json; +use crate::lineage_command::models::{LineageRendering, SemanticUse}; + +/// The rendering, followed by direct semantic uses when they were requested. +pub fn lineage_stdout( + rendering: LineageRendering, + uses: Option<&[SemanticUse]>, +) -> Result { + let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))); + match (rendering, uses) { + (LineageRendering::Json(value), None) => Ok(format!("{}\n", dumps(&value, &dialect)?)), + (LineageRendering::Text(text), None) => Ok(format!("\n{text}\n\n")), + (LineageRendering::Json(value), Some(uses)) => { + let mut entries: Vec<(String, JsonValue)> = match value { + JsonValue::Object(entries) => entries, + _ => Vec::new(), + }; + entries.push(( + "semantic_uses".to_owned(), + JsonValue::Array(uses.iter().map(use_json).collect()), + )); + Ok(format!( + "{}\n", + dumps(&JsonValue::Object(entries), &dialect)? + )) + } + (LineageRendering::Text(text), Some(uses)) => { + let mut lines: Vec = vec![ + String::new(), + text, + String::new(), + "Direct semantic column uses".to_owned(), + ]; + if uses.is_empty() { + lines.push(" none".to_owned()); + } + lines.extend(uses.iter().map(|semantic_use| { + format!( + " {} [{}] {}:{}.{} <- {}", + semantic_use.consumer_model, + semantic_use.context, + semantic_use.source.resource_type, + semantic_use.source.resource_name, + semantic_use.source.column_name, + semantic_use.expression + ) + })); + Ok(format!("{}\n\n", lines.join("\n"))) + } + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/mod.rs b/crates/sqlbuild-analysis/src/lineage_command/main/mod.rs new file mode 100644 index 0000000000..c6b61af381 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/mod.rs @@ -0,0 +1,12 @@ +pub mod column_candidates; +pub mod lineage_path_nodes; +pub mod lineage_stdout; +pub mod normalize_lineage_target; +pub mod parse_lineage_depth; +pub mod render_column_trace; +pub mod render_lineage_graph; +pub mod resolve_lineage_target; +pub mod select_anchored_lineage; +pub mod select_directional_lineage; +pub mod select_target_lineage; +pub mod validate_lineage_request; diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/normalize_lineage_target.rs b/crates/sqlbuild-analysis/src/lineage_command/main/normalize_lineage_target.rs new file mode 100644 index 0000000000..05cdc7b5c4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/normalize_lineage_target.rs @@ -0,0 +1,21 @@ +//! A lineage target without its printed kind prefix. + +use crate::lineage_command::_helpers::targets::unknown_lineage_target; +use crate::lineage_command::errors::LineageError; +use crate::lineage_command::main::resolve_lineage_target::resolve_lineage_target; +use crate::lineage_command::models::{LineageKey, LineageTarget}; + +/// Strip a kind prefix such as `model:` after checking it matches the resource. +pub fn normalize_lineage_target( + target: &str, + lookup: &dyn Fn(&str) -> Option, +) -> Result { + let resolved: LineageTarget = resolve_lineage_target(target, lookup); + if resolved.kind.is_none() { + return Ok(target.to_owned()); + } + if !resolved.kind_matches { + return Err(unknown_lineage_target(target)); + } + Ok(resolved.name) +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/parse_lineage_depth.rs b/crates/sqlbuild-analysis/src/lineage_command/main/parse_lineage_depth.rs new file mode 100644 index 0000000000..331b47ba2e --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/parse_lineage_depth.rs @@ -0,0 +1,15 @@ +//! `--depth`: a non-negative integer, or `all` for unlimited traversal. + +use crate::lineage_command::constants::{DEPTH_MESSAGE, UNLIMITED_DEPTH_VALUE}; +use crate::lineage_command::errors::LineageError; + +/// The depth limit, or `None` for `all`. +pub fn parse_lineage_depth(raw: &str) -> Result, LineageError> { + if raw == UNLIMITED_DEPTH_VALUE { + return Ok(None); + } + match raw.trim().parse::() { + Ok(depth) if depth >= 0 => Ok(Some(depth)), + _ => Err(LineageError::new("C304", DEPTH_MESSAGE)), + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/render_column_trace.rs b/crates/sqlbuild-analysis/src/lineage_command/main/render_column_trace.rs new file mode 100644 index 0000000000..df71aca494 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/render_column_trace.rs @@ -0,0 +1,22 @@ +//! A column trace in the requested output format. + +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::lineage_command::_helpers::column_text::{column_list, column_tree}; +use crate::lineage_command::_helpers::json::trace_json; +use crate::lineage_command::constants::{JSON_OUTPUT_FORMAT, LIST_OUTPUT_FORMAT}; +use crate::lineage_command::models::{ColumnTrace, LineageRendering}; + +/// JSON, a dependency list, or a tree from the traced column. +#[must_use] +pub fn render_column_trace( + trace: &ColumnTrace, + output_format: &str, + style: CliStyle, +) -> LineageRendering { + match output_format { + JSON_OUTPUT_FORMAT => LineageRendering::Json(trace_json(trace)), + LIST_OUTPUT_FORMAT => LineageRendering::Text(column_list(trace, style)), + _ => LineageRendering::Text(column_tree(trace, style)), + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/render_lineage_graph.rs b/crates/sqlbuild-analysis/src/lineage_command/main/render_lineage_graph.rs new file mode 100644 index 0000000000..51786ec3f2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/render_lineage_graph.rs @@ -0,0 +1,22 @@ +//! A selected relation graph in the requested output format. + +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::lineage_command::_helpers::graph_text::{graph_list, graph_tree}; +use crate::lineage_command::_helpers::json::graph_json; +use crate::lineage_command::constants::{JSON_OUTPUT_FORMAT, LIST_OUTPUT_FORMAT}; +use crate::lineage_command::models::{LineageGraph, LineageRendering}; + +/// JSON, an edge list, or a tree around the focus node. +#[must_use] +pub fn render_lineage_graph( + graph: &LineageGraph, + output_format: &str, + style: CliStyle, +) -> LineageRendering { + match output_format { + JSON_OUTPUT_FORMAT => LineageRendering::Json(graph_json(graph)), + LIST_OUTPUT_FORMAT => LineageRendering::Text(graph_list(graph, style)), + _ => LineageRendering::Text(graph_tree(graph, style)), + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/resolve_lineage_target.rs b/crates/sqlbuild-analysis/src/lineage_command/main/resolve_lineage_target.rs new file mode 100644 index 0000000000..fd58503a22 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/resolve_lineage_target.rs @@ -0,0 +1,43 @@ +//! Read `[kind:]` or `[kind:].` against project resource names. + +use crate::lineage_command::constants::{ + COLUMN_TARGET_KIND, COLUMN_TARGET_SEPARATOR, KIND_SEPARATOR, MODEL_RESOURCE_TYPE, TARGET_KINDS, +}; +use crate::lineage_command::models::{LineageKey, LineageTarget}; + +/// The target's written kind, name, resource and column, and whether the kind fits the resource. +pub fn resolve_lineage_target( + target: &str, + lookup: &dyn Fn(&str) -> Option, +) -> LineageTarget { + let written_kind: Option<&str> = target + .split_once(KIND_SEPARATOR) + .map(|(kind, _)| kind) + .filter(|kind| TARGET_KINDS.contains(kind)); + let name: &str = match written_kind { + Some(kind) => &target[kind.len() + 1..], + None => target, + }; + let (resource_name, column_name): (&str, Option<&str>) = + match name.split_once(COLUMN_TARGET_SEPARATOR) { + Some((resource, column)) => (resource, Some(column)), + None => (name, None), + }; + let key: Option = lookup(resource_name); + let kind_matches: bool = match written_kind { + None => true, + Some(COLUMN_TARGET_KIND) => { + key.as_ref() + .is_some_and(|key| key.resource_type == MODEL_RESOURCE_TYPE) + && column_name.is_some_and(|column| !column.is_empty()) + } + Some(kind) => key.as_ref().is_some_and(|key| key.resource_type == kind), + }; + LineageTarget { + kind: written_kind.map(str::to_owned), + name: name.to_owned(), + key, + column_name: column_name.map(str::to_owned), + kind_matches, + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/select_anchored_lineage.rs b/crates/sqlbuild-analysis/src/lineage_command/main/select_anchored_lineage.rs new file mode 100644 index 0000000000..cafa3afa31 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/select_anchored_lineage.rs @@ -0,0 +1,33 @@ +//! Selector lineage focused on its anchors, trimmed to `--depth` around them. + +use std::collections::HashSet; + +use crate::lineage_command::_helpers::graph::{build_graph, closure, edge_map, sorted_keys}; +use crate::lineage_command::models::{LineageAnchors, LineageGraph, LineageIndex, LineageKey}; + +/// The selected keys, keeping only those within `depth` of an anchor when a depth is given. +pub fn select_anchored_lineage( + index: &LineageIndex, + selected: &[LineageKey], + (anchors, depth): (&LineageAnchors, Option), +) -> LineageGraph { + let mut keys: HashSet = selected.iter().cloned().collect(); + if let Some(depth) = depth { + let mut retained: HashSet = anchors.retained.iter().cloned().collect(); + retained.extend(anchors.upstream.iter().cloned()); + retained.extend(anchors.downstream.iter().cloned()); + retained.extend(closure( + &edge_map(&index.upstream), + &anchors.upstream, + Some(depth), + )); + retained.extend(closure( + &edge_map(&index.downstream), + &anchors.downstream, + Some(depth), + )); + keys.retain(|key| retained.contains(key)); + } + let focus: Vec = sorted_keys(anchors.upstream.iter().chain(&anchors.downstream)); + build_graph(index, &keys, (focus, None)) +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/select_directional_lineage.rs b/crates/sqlbuild-analysis/src/lineage_command/main/select_directional_lineage.rs new file mode 100644 index 0000000000..9c0af1b2a4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/select_directional_lineage.rs @@ -0,0 +1,20 @@ +//! Selector lineage expanded by an explicit `--direction`. + +use std::collections::HashSet; + +use crate::lineage_command::_helpers::graph::{build_graph, expand, sorted_keys}; +use crate::lineage_command::models::{LineageGraph, LineageIndex, LineageKey}; + +/// Selected keys plus what `direction` reaches within `depth`, minus the excluded keys. +pub fn select_directional_lineage( + index: &LineageIndex, + (selected, excluded): (&[LineageKey], &[LineageKey]), + (direction, depth): (&str, Option), +) -> LineageGraph { + let excluded: HashSet<&LineageKey> = excluded.iter().collect(); + let mut keys: HashSet = selected.iter().cloned().collect(); + keys.extend(expand(index, selected, (direction, depth))); + keys.retain(|key| !excluded.contains(key)); + let focus: Vec = sorted_keys(selected.iter().filter(|key| !excluded.contains(key))); + build_graph(index, &keys, (focus, Some(direction))) +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/select_target_lineage.rs b/crates/sqlbuild-analysis/src/lineage_command/main/select_target_lineage.rs new file mode 100644 index 0000000000..9f5a8aaf63 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/select_target_lineage.rs @@ -0,0 +1,35 @@ +//! Relation lineage around positional targets. + +use std::collections::{HashMap, HashSet}; + +use crate::lineage_command::_helpers::graph::{build_graph, expand, sorted_keys}; +use crate::lineage_command::_helpers::targets::unknown_lineage_target; +use crate::lineage_command::errors::LineageError; +use crate::lineage_command::models::{LineageGraph, LineageIndex, LineageKey}; + +/// The targets and every resource `direction` reaches from them within `depth`. +pub fn select_target_lineage( + index: &LineageIndex, + targets: &[String], + (direction, depth): (&str, Option), +) -> Result { + let names: HashMap<&str, &LineageKey> = index + .all_keys + .iter() + .map(|(name, key)| (name.as_str(), key)) + .collect(); + let mut keys: Vec = Vec::new(); + for target in targets { + let Some(key) = names.get(target.as_str()) else { + return Err(unknown_lineage_target(target)); + }; + keys.push((*key).clone()); + } + let mut selected: HashSet = keys.iter().cloned().collect(); + selected.extend(expand(index, &keys, (direction, depth))); + Ok(build_graph( + index, + &selected, + (sorted_keys(&keys), Some(direction)), + )) +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/main/validate_lineage_request.rs b/crates/sqlbuild-analysis/src/lineage_command/main/validate_lineage_request.rs new file mode 100644 index 0000000000..1cf72a8b9c --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/main/validate_lineage_request.rs @@ -0,0 +1,40 @@ +//! The combinations of targets, `--select` and `--exclude` `sqb lineage` accepts. + +use crate::lineage_command::constants::COLUMN_TARGET_SEPARATOR; +use crate::lineage_command::errors::LineageError; +use crate::lineage_command::models::LineageRequestShape; + +/// Reject conflicting or missing target and selector options. +pub fn validate_lineage_request(request: &LineageRequestShape) -> Result<(), LineageError> { + if !request.targets.is_empty() && !request.select.is_empty() { + return Err(LineageError::new( + "C301", + "lineage accepts either targets or --select, not both", + )); + } + if request.targets.is_empty() && request.select.is_empty() { + return Err(LineageError::new( + "C302", + "lineage requires a target or --select", + )); + } + if request.targets.len() > 1 + && request + .targets + .iter() + .any(|target| target.contains(COLUMN_TARGET_SEPARATOR)) + { + return Err(LineageError::new( + "C320", + "column lineage accepts exactly one model.column target", + ) + .with_help("run one lineage command per column")); + } + if !request.exclude.is_empty() && request.select.is_empty() { + return Err(LineageError::new( + "C303", + "--exclude can only be used with --select", + )); + } + Ok(()) +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/mod.rs b/crates/sqlbuild-analysis/src/lineage_command/mod.rs new file mode 100644 index 0000000000..43006b71a6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/mod.rs @@ -0,0 +1,10 @@ +//! `sqb lineage`: target and depth parsing, relation selection and text and JSON output. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/lineage_command/models.rs b/crates/sqlbuild-analysis/src/lineage_command/models.rs new file mode 100644 index 0000000000..07d2daada8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/models.rs @@ -0,0 +1,119 @@ +//! Relation lineage indexes, selected graphs, column traces and semantic uses. + +use std::collections::HashMap; + +/// A project resource: its type as Python's `str()` renders it, and its name. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct LineageKey { + pub resource_type: String, + pub name: String, +} + +/// One displayable resource. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LineageNode { + pub key: LineageKey, + pub relative_path: Option, + pub qualified_name: Option, +} + +/// The structural project graph relation lineage reads, every collection in Python dict order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LineageIndex { + pub nodes: HashMap, + pub upstream: Vec<(LineageKey, Vec)>, + pub downstream: Vec<(LineageKey, Vec)>, + pub tags: Vec<(String, Vec)>, + pub paths: Vec<(LineageKey, String)>, + pub all_keys: Vec<(String, LineageKey)>, +} + +/// A selected slice of the relation graph. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LineageGraph { + pub nodes: Vec, + pub edges: Vec<(LineageKey, LineageKey)>, + pub focus_keys: Vec, + pub direction: Option, +} + +/// One positional target, `[kind:]` or `[kind:].`, resolved by name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LineageTarget { + pub kind: Option, + pub name: String, + pub key: Option, + pub column_name: Option, + pub kind_matches: bool, +} + +/// Selector anchors depth trimming starts from. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LineageAnchors { + pub upstream: Vec, + pub downstream: Vec, + pub retained: Vec, +} + +/// A resource-qualified column. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LineageColumn { + pub resource_type: String, + pub resource_name: String, + pub column_name: String, +} + +/// One column lineage edge. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LineageColumnEdge { + pub source: LineageColumn, + pub target: LineageColumn, + pub transform: String, + pub confidence: String, +} + +/// A traced column and the edges reached from it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ColumnTrace { + pub target: LineageColumn, + pub trace: Vec, + pub direction: String, + pub mode: String, + pub max_depth: Option, + pub analyzed_model_count: usize, + pub truncated: bool, +} + +/// The models a column trace analyses, and whether the depth limit hid more. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ColumnCandidates { + pub model_names: Vec, + pub truncated: bool, +} + +/// One direct non-projection use of a resource column. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SemanticUse { + pub consumer_model: String, + pub context: String, + pub expression: String, + pub source: LineageColumn, + pub confidence: String, + pub line: Option, + pub column: Option, +} + +/// The command options request validation reads. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LineageRequestShape { + pub targets: Vec, + pub select: Vec, + pub exclude: Vec, +} + +/// Rendered lineage before printing: JSON stays a value so semantic uses can join it. +#[derive(Clone, Debug, PartialEq)] +pub enum LineageRendering { + Json(sqlbuild_core::json::models::JsonValue), + Text(String), +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/tests.rs b/crates/sqlbuild-analysis/src/lineage_command/tests.rs new file mode 100644 index 0000000000..b60599304f --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_lineage_command.rs"] +mod lineage_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/lineage_command/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage_command/tests/helpers.rs new file mode 100644 index 0000000000..b37c7758dc --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/tests/helpers.rs @@ -0,0 +1,55 @@ +use std::collections::HashMap; + +use crate::lineage_command::models::{LineageIndex, LineageKey, LineageNode}; + +/// A key of `resource_type` named `name`. +pub(super) fn key(resource_type: &str, name: &str) -> LineageKey { + LineageKey { + resource_type: resource_type.to_owned(), + name: name.to_owned(), + } +} + +/// `raw_orders -> stg_orders -> fact_orders <- waffle_types`, with model paths. +pub(super) fn orders_index() -> LineageIndex { + let raw: LineageKey = key("source", "raw_orders"); + let stg: LineageKey = key("model", "stg_orders"); + let fact: LineageKey = key("model", "fact_orders"); + let seed: LineageKey = key("seed", "waffle_types"); + let nodes: HashMap = [ + (stg.clone(), Some("models/stg_orders.sql")), + (fact.clone(), Some("models/fact_orders.sql")), + (raw.clone(), None), + (seed.clone(), None), + ] + .into_iter() + .map(|(key, path)| { + ( + key.clone(), + LineageNode { + key, + relative_path: path.map(str::to_owned), + qualified_name: None, + }, + ) + }) + .collect(); + LineageIndex { + nodes, + upstream: vec![ + (stg.clone(), vec![raw.clone()]), + (fact.clone(), vec![stg.clone(), seed.clone()]), + ], + downstream: vec![ + (raw.clone(), vec![stg.clone()]), + (stg.clone(), vec![fact.clone()]), + (seed.clone(), vec![fact.clone()]), + ], + tags: Vec::new(), + paths: Vec::new(), + all_keys: [raw, stg, fact, seed] + .into_iter() + .map(|key| (key.name.clone(), key)) + .collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/tests/test_lineage_command.rs b/crates/sqlbuild-analysis/src/lineage_command/tests/test_lineage_command.rs new file mode 100644 index 0000000000..c1173a53a4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/tests/test_lineage_command.rs @@ -0,0 +1,133 @@ +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::lineage_command::main::lineage_stdout::lineage_stdout; +use crate::lineage_command::main::normalize_lineage_target::normalize_lineage_target; +use crate::lineage_command::main::render_lineage_graph::render_lineage_graph; +use crate::lineage_command::main::select_target_lineage::select_target_lineage; +use crate::lineage_command::models::{LineageIndex, LineageKey}; +use crate::lineage_command::tests::helpers::orders_index; +use crate::lineage_command::tests::test_types::{ResolveTargetTestCase, TargetLineageTestCase}; + +#[test] +fn given_targets_when_selecting_lineage_then_stdout_matches() { + let test_cases = [ + TargetLineageTestCase { + description: "an upstream tree sorts children by type and name", + targets: vec!["fact_orders".to_owned()], + output_format: "tree", + direction: "upstream", + depth: None, + expected_stdout: Ok( + "\nLineage model fact_orders models/fact_orders.sql upstream\n\ + \u{251c}\u{2500}\u{2500} model stg_orders models/stg_orders.sql\n\ + \u{2502} \u{2514}\u{2500}\u{2500} source raw_orders\n\ + \u{2514}\u{2500}\u{2500} seed waffle_types\n\n", + ), + }, + TargetLineageTestCase { + description: "a depth-limited downstream edge list aligns arrows", + targets: vec!["raw_orders".to_owned(), "waffle_types".to_owned()], + output_format: "list", + direction: "downstream", + depth: Some(1), + expected_stdout: Ok("\nmodel:stg_orders -> model:fact_orders\n\ + seed:waffle_types -> model:fact_orders\n\ + source:raw_orders -> model:stg_orders\n\n"), + }, + TargetLineageTestCase { + description: "JSON lists nodes, edges, focus and direction", + targets: vec!["stg_orders".to_owned()], + output_format: "json", + direction: "downstream", + depth: None, + expected_stdout: Ok( + "{\n \"nodes\": [\n {\n \"id\": \"model:fact_orders\",\n \ + \"name\": \"fact_orders\",\n \"resource_type\": \"model\",\n \ + \"relative_path\": \"models/fact_orders.sql\"\n },\n {\n \ + \"id\": \"model:stg_orders\",\n \"name\": \"stg_orders\",\n \ + \"resource_type\": \"model\",\n \"relative_path\": \"models/stg_orders.sql\"\n \ + }\n ],\n \"edges\": [\n {\n \"from\": \"model:stg_orders\",\n \ + \"to\": \"model:fact_orders\"\n }\n ],\n \"focus\": [\n \ + \"model:stg_orders\"\n ],\n \"direction\": \"downstream\"\n}\n", + ), + }, + TargetLineageTestCase { + description: "an unknown target is a user error", + targets: vec!["stg_orders+".to_owned()], + output_format: "tree", + direction: "upstream", + depth: None, + expected_stdout: Err("C305"), + }, + ]; + + for test_case in test_cases { + let index: LineageIndex = orders_index(); + let observed: Result = select_target_lineage( + &index, + &test_case.targets, + (test_case.direction, test_case.depth), + ) + .map_err(|error| error.code) + .map(|graph| { + lineage_stdout( + render_lineage_graph( + &graph, + test_case.output_format, + CliStyle { use_color: false }, + ), + None, + ) + .unwrap_or_default() + }); + assert_eq!( + observed, + test_case.expected_stdout.map(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_prefixed_target_when_normalizing_then_kind_must_match() { + let test_cases = [ + ResolveTargetTestCase { + description: "a matching prefix is stripped", + target: "model:stg_orders", + expected_name: Ok("stg_orders"), + }, + ResolveTargetTestCase { + description: "a column prefix keeps the column", + target: "column:stg_orders.order_id", + expected_name: Ok("stg_orders.order_id"), + }, + ResolveTargetTestCase { + description: "an unprefixed target is returned unchanged", + target: "anything.at_all", + expected_name: Ok("anything.at_all"), + }, + ResolveTargetTestCase { + description: "a mismatched prefix is an unknown target", + target: "seed:stg_orders", + expected_name: Err("C305"), + }, + ]; + + for test_case in test_cases { + let index: LineageIndex = orders_index(); + let lookup = |name: &str| -> Option { + index + .all_keys + .iter() + .find(|(candidate, _)| candidate == name) + .map(|(_, key)| key.clone()) + }; + assert_eq!( + normalize_lineage_target(test_case.target, &lookup).map_err(|error| error.code), + test_case.expected_name.map(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lineage_command/tests/test_types.rs b/crates/sqlbuild-analysis/src/lineage_command/tests/test_types.rs new file mode 100644 index 0000000000..80a09cf946 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage_command/tests/test_types.rs @@ -0,0 +1,14 @@ +pub(super) struct TargetLineageTestCase { + pub(super) description: &'static str, + pub(super) targets: Vec, + pub(super) output_format: &'static str, + pub(super) direction: &'static str, + pub(super) depth: Option, + pub(super) expected_stdout: Result<&'static str, &'static str>, +} + +pub(super) struct ResolveTargetTestCase { + pub(super) description: &'static str, + pub(super) target: &'static str, + pub(super) expected_name: Result<&'static str, &'static str>, +} diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 931992c006..450b6d7bf1 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -100,7 +100,7 @@ const SCOPE_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const SCOPE_ROUTE: Route = routed( "run_scope", - ("sqlbuild.cli.commands.main.inspection._scope", "run_scope"), + ("sqlbuild._native", "run_scope_command"), &SCOPE_ARGUMENTS, REQUEST_KEYWORD_SHAPE, ); @@ -215,12 +215,9 @@ const FRESHNESS_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const FRESHNESS_ROUTE: Route = routed( "run_freshness", - ( - "sqlbuild.cli.commands.main.execution._freshness", - "run_freshness", - ), + ("sqlbuild._native", "run_freshness_command"), &FRESHNESS_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const TEST_FIELDS: [Field; 12] = [ @@ -365,12 +362,9 @@ const LINEAGE_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const LINEAGE_ROUTE: Route = routed( "run_lineage", - ( - "sqlbuild.cli.commands.main.inspection._lineage", - "run_lineage", - ), + ("sqlbuild._native", "run_lineage_command"), &LINEAGE_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const CLONE_FIELDS: [Field; 12] = [ @@ -500,7 +494,7 @@ const DEBUG_ARGUMENTS: [Field; 5] = [ ]; pub(crate) const DEBUG_ROUTE: Route = routed( "run_debug", - ("sqlbuild.cli.commands.main.inspection._debug", "run_debug"), + ("sqlbuild._native", "run_debug_command"), &DEBUG_ARGUMENTS, CallShape { slot_positional: 0, @@ -527,9 +521,9 @@ const JANITOR_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const JANITOR_ROUTE: Route = routed( "run_janitor", - ("sqlbuild.cli.commands.main.state._janitor", "run_janitor"), + ("sqlbuild._native", "run_janitor_command"), &JANITOR_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const REFACTOR_FIELDS: [Field; 9] = [ @@ -551,10 +545,7 @@ pub(crate) const REFACTOR_ROUTE: Route = Route { unavailable_code: Some("C950"), ..routed( "run_refactor", - ( - "sqlbuild.cli.commands.main.project._refactor", - "run_refactor", - ), + ("sqlbuild._native", "run_refactor_command"), &REFACTOR_ARGUMENTS, REQUEST_KEYWORD_SHAPE, ) @@ -654,10 +645,7 @@ const PLAYGROUND_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const PLAYGROUND_ROUTE: Route = routed( "run_playground", - ( - "sqlbuild.cli.commands.main.workspace._playground", - "run_playground", - ), + ("sqlbuild._native", "run_playground_command"), &PLAYGROUND_ARGUMENTS, REQUEST_SHAPE, ); @@ -670,10 +658,7 @@ const SKILLS_ARGUMENTS: [Field; 4] = [ ]; pub(crate) const SKILLS_ROUTE: Route = routed( "run_skills_update", - ( - "sqlbuild.cli.commands.main.workspace._skills", - "run_skills_update", - ), + ("sqlbuild._native", "run_skills_command"), &SKILLS_ARGUMENTS, CallShape { slot_positional: 0, diff --git a/crates/sqlbuild-config/src/command_connection/_helpers/identifiers.rs b/crates/sqlbuild-config/src/command_connection/_helpers/identifiers.rs new file mode 100644 index 0000000000..aca0c4de16 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/_helpers/identifiers.rs @@ -0,0 +1,31 @@ +//! Snowflake warehouse identifier syntax. + +use crate::command_connection::constants::MIN_QUOTED_IDENTIFIER_LENGTH; + +/// Whether `text` fully matches `[A-Za-z_][A-Za-z0-9_$]*|"(?:[^"]|"")+"`. +pub(crate) fn is_warehouse_identifier(text: &str) -> bool { + let mut characters = text.chars(); + let Some(first) = characters.next() else { + return false; + }; + if first.is_ascii_alphabetic() || first == '_' { + return characters.all(|character| { + character.is_ascii_alphanumeric() || character == '_' || character == '$' + }); + } + if first != '"' || text.len() < MIN_QUOTED_IDENTIFIER_LENGTH || !text.ends_with('"') { + return false; + } + let interior: &str = &text[1..text.len() - 1]; + let mut quote_run: usize = 0; + for character in interior.chars() { + if character == '"' { + quote_run += 1; + } else if !quote_run.is_multiple_of(2) { + return false; + } else { + quote_run = 0; + } + } + quote_run.is_multiple_of(2) +} diff --git a/crates/sqlbuild-config/src/command_connection/_helpers/mod.rs b/crates/sqlbuild-config/src/command_connection/_helpers/mod.rs new file mode 100644 index 0000000000..bd9fd2a0df --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod identifiers; diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs new file mode 100644 index 0000000000..3fd864dcbd --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -0,0 +1,24 @@ +//! Command connection sources, keys and wording. + +pub const CLI_WAREHOUSE_SOURCE: &str = "--warehouse"; +pub const CONNECTION_WAREHOUSE_SOURCE: &str = "connection warehouse"; +pub const CONNECTION_DEFAULT_WAREHOUSE_SOURCE: &str = "connection default"; +pub const CONNECTION_WAREHOUSE_KEY: &str = "warehouse"; +pub const CONNECTION_DATABASE_KEY: &str = "database"; +pub const CONNECTION_SOURCE_KEY: &str = "source"; +pub const DBT_PROFILE_CONNECTION_SOURCE: &str = "dbt_profile"; +pub const DUCKDB_ADAPTER: &str = "duckdb"; +pub const DUCKDB_MEMORY_DATABASE: &str = ":memory:"; +pub const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +pub const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; +/// Connection keys a dbt profile connection routes on rather than connects with. +pub const DBT_PROFILE_ROUTING_KEYS: [&str; 5] = + ["source", "profile", "target", "project_dir", "profiles_dir"]; +/// Keys that suggest a Snowflake connection was given to the DuckDB adapter. +pub(crate) const SNOWFLAKE_LIKE_KEYS: [&str; 5] = + ["account", "authenticator", "role", "token", "warehouse"]; +pub(crate) const WAREHOUSE_ERROR_CODE: &str = "C261"; +pub(crate) const UNSUPPORTED_WAREHOUSE_CODE: &str = "C260"; +pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; +/// Two quotes around at least one character. +pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; diff --git a/crates/sqlbuild-config/src/command_connection/errors.rs b/crates/sqlbuild-config/src/command_connection/errors.rs new file mode 100644 index 0000000000..42c42996e4 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/errors.rs @@ -0,0 +1,24 @@ +//! Failures resolving a command's connection or warehouse. + +/// A command connection failure, raised as the Python exception its kind names. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CommandConnectionError { + /// `CliUserError(message, code=code, help=help)`. + User { + code: Option<&'static str>, + message: String, + help: Option, + }, + /// `ProjectConfigError(message)`. + ProjectConfig(String), +} + +impl CommandConnectionError { + /// The message the raised exception carries. + #[must_use] + pub fn message(&self) -> &str { + match self { + Self::User { message, .. } | Self::ProjectConfig(message) => message, + } + } +} diff --git a/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs b/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs new file mode 100644 index 0000000000..450078ba96 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs @@ -0,0 +1,24 @@ +//! The authored warehouse a target sets for a command group. + +use crate::command_connection::constants::{LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME}; +use crate::command_connection::models::AuthoredWarehouse; + +/// The local target's group warehouse, else the project target's, with the file that sets it. +#[must_use] +pub fn authored_warehouse( + (target_name, group): (&str, &str), + local_value: Option, + project_value: Option, +) -> Option { + let (warehouse, file_name) = match (local_value, project_value) { + (Some(local), _) => (local, LOCAL_CONFIG_FILENAME), + (None, Some(project)) => (project, PROJECT_CONFIG_FILENAME), + (None, None) => return None, + }; + Some(AuthoredWarehouse { + warehouse, + target_name: target_name.to_owned(), + group: group.to_owned(), + file_name: file_name.to_owned(), + }) +} diff --git a/crates/sqlbuild-config/src/command_connection/main/check_cli_warehouse.rs b/crates/sqlbuild-config/src/command_connection/main/check_cli_warehouse.rs new file mode 100644 index 0000000000..234defb964 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/check_cli_warehouse.rs @@ -0,0 +1,33 @@ +//! Validate a `--warehouse` value. + +use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::command_connection::_helpers::identifiers::is_warehouse_identifier; +use crate::command_connection::constants::WAREHOUSE_ERROR_CODE; +use crate::command_connection::errors::CommandConnectionError; + +/// Reject an empty or malformed `--warehouse` identifier. +pub fn check_cli_warehouse(cli_warehouse: &str) -> Result<(), CommandConnectionError> { + if python_strip(cli_warehouse).is_empty() { + return Err(CommandConnectionError::User { + code: Some(WAREHOUSE_ERROR_CODE), + message: "--warehouse requires a warehouse name".to_owned(), + help: Some("pass a name such as `--warehouse ADHOC_WH`, or omit the flag".to_owned()), + }); + } + if is_warehouse_identifier(cli_warehouse) { + return Ok(()); + } + Err(CommandConnectionError::User { + code: Some(WAREHOUSE_ERROR_CODE), + message: format!( + "--warehouse {} is not a valid Snowflake warehouse identifier", + python_str_repr(cli_warehouse) + ), + help: Some( + "pass an unquoted identifier such as `--warehouse ADHOC_WH`, or a double-quoted identifier such as `--warehouse '\"Adhoc WH\"'`" + .to_owned(), + ), + }) +} diff --git a/crates/sqlbuild-config/src/command_connection/main/check_expanded_warehouse.rs b/crates/sqlbuild-config/src/command_connection/main/check_expanded_warehouse.rs new file mode 100644 index 0000000000..f0037e8f35 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/check_expanded_warehouse.rs @@ -0,0 +1,32 @@ +//! Validate a target group's warehouse after template expansion. + +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::command_connection::_helpers::identifiers::is_warehouse_identifier; +use crate::command_connection::constants::{SETTING_SNIPPET_INDENT, WAREHOUSE_ERROR_CODE}; +use crate::command_connection::errors::CommandConnectionError; +use crate::command_connection::models::AuthoredWarehouse; + +/// Reject an expanded group warehouse that is not a Snowflake identifier. +pub fn check_expanded_warehouse( + authored: &AuthoredWarehouse, + warehouse: &str, +) -> Result<(), CommandConnectionError> { + if is_warehouse_identifier(warehouse) { + return Ok(()); + } + let section: String = format!("targets.{}.warehouses", authored.target_name); + Err(CommandConnectionError::User { + code: Some(WAREHOUSE_ERROR_CODE), + message: format!( + "{} [{section}] {} resolves to {}, which is not a valid Snowflake warehouse identifier", + authored.file_name, + authored.group, + python_str_repr(warehouse) + ), + help: Some(format!( + "to use an unquoted identifier (or a double-quoted one such as \"Adhoc WH\"), set this in {}:\n{SETTING_SNIPPET_INDENT}[{section}]\n{SETTING_SNIPPET_INDENT}{} = \"ADHOC_WH\"", + authored.file_name, authored.group + )), + }) +} diff --git a/crates/sqlbuild-config/src/command_connection/main/check_preview_target.rs b/crates/sqlbuild-config/src/command_connection/main/check_preview_target.rs new file mode 100644 index 0000000000..c5b3b2bb5c --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/check_preview_target.rs @@ -0,0 +1,27 @@ +//! Validate an inspection-only `--as` target. + +use crate::command_connection::errors::CommandConnectionError; + +/// Reject an `--as` target that no project or local config defines. +pub fn check_preview_target( + as_target: &str, + command_name: &str, + configured: &[String], +) -> Result<(), CommandConnectionError> { + if configured.iter().any(|target| target == as_target) { + return Ok(()); + } + let mut names: Vec<&str> = configured.iter().map(String::as_str).collect(); + names.sort_unstable(); + names.dedup(); + let listed: String = if names.is_empty() { + "none".to_owned() + } else { + names.join(", ") + }; + Err(CommandConnectionError::User { + code: None, + message: format!("unknown target '{as_target}' for {command_name} --as"), + help: Some(format!("Configured targets: {listed}.")), + }) +} diff --git a/crates/sqlbuild-config/src/command_connection/main/missing_connection_error.rs b/crates/sqlbuild-config/src/command_connection/main/missing_connection_error.rs new file mode 100644 index 0000000000..a54181c38d --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/missing_connection_error.rs @@ -0,0 +1,19 @@ +//! The error for a warehouse command with no configured connection. + +use crate::command_connection::errors::CommandConnectionError; + +/// The error naming `project_file` and, when one is selected, the target without a connection. +#[must_use] +pub fn missing_connection_error( + project_file: &str, + target_name: Option<&str>, +) -> CommandConnectionError { + CommandConnectionError::ProjectConfig(match target_name { + None => format!( + "{project_file} no target is selected and no top-level [connection] is configured; add [connections.] and a [targets.] that uses it, then set default_target or pass --target" + ), + Some(target) => format!( + "{project_file} target {target} has no connection; add [connections.] (or set connection = \"\" on [targets.{target}]) in sqlbuild_project.toml or sqlbuild_local.toml" + ), + }) +} diff --git a/crates/sqlbuild-config/src/command_connection/main/mod.rs b/crates/sqlbuild-config/src/command_connection/main/mod.rs new file mode 100644 index 0000000000..0cfa5f9805 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/mod.rs @@ -0,0 +1,7 @@ +pub mod authored_warehouse; +pub mod check_cli_warehouse; +pub mod check_expanded_warehouse; +pub mod check_preview_target; +pub mod missing_connection_error; +pub mod snowflake_key_warning; +pub mod unsupported_cli_warehouse; diff --git a/crates/sqlbuild-config/src/command_connection/main/snowflake_key_warning.rs b/crates/sqlbuild-config/src/command_connection/main/snowflake_key_warning.rs new file mode 100644 index 0000000000..f3b340ea03 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/snowflake_key_warning.rs @@ -0,0 +1,24 @@ +//! The warning for Snowflake-like keys in a DuckDB connection. + +use crate::command_connection::constants::{DUCKDB_ADAPTER, SNOWFLAKE_LIKE_KEYS}; + +/// The warning line, when the DuckDB adapter got Snowflake-like connection keys. +#[must_use] +pub fn snowflake_key_warning(adapter_name: &str, keys: &[String]) -> Option { + if adapter_name != DUCKDB_ADAPTER { + return None; + } + let mut suspicious: Vec<&str> = keys + .iter() + .map(String::as_str) + .filter(|key| SNOWFLAKE_LIKE_KEYS.contains(key)) + .collect(); + if suspicious.is_empty() { + return None; + } + suspicious.sort_unstable(); + Some(format!( + "Warning: DuckDB adapter is active, but connection contains Snowflake-like keys: {}. If this is a Snowflake local config, add `adapter = \"snowflake\"` at the top of sqlbuild_local.toml, before any section.", + suspicious.join(", ") + )) +} diff --git a/crates/sqlbuild-config/src/command_connection/main/unsupported_cli_warehouse.rs b/crates/sqlbuild-config/src/command_connection/main/unsupported_cli_warehouse.rs new file mode 100644 index 0000000000..62a63fff22 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/unsupported_cli_warehouse.rs @@ -0,0 +1,22 @@ +//! `--warehouse` for an adapter without a session warehouse. + +use crate::command_connection::constants::UNSUPPORTED_WAREHOUSE_CODE; +use crate::command_connection::errors::CommandConnectionError; + +/// The error for a `--warehouse` the adapter cannot select. +#[must_use] +pub fn unsupported_cli_warehouse( + cli_warehouse: &str, + adapter_name: &str, +) -> CommandConnectionError { + CommandConnectionError::User { + code: Some(UNSUPPORTED_WAREHOUSE_CODE), + message: format!( + "--warehouse {cli_warehouse} was given, but the '{adapter_name}' adapter has no warehouse to select" + ), + help: Some( + "remove --warehouse; it needs an adapter with a session warehouse, such as snowflake or an adapter extending it" + .to_owned(), + ), + } +} diff --git a/crates/sqlbuild-config/src/command_connection/mod.rs b/crates/sqlbuild-config/src/command_connection/mod.rs new file mode 100644 index 0000000000..899237ebcb --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/command_connection/models.rs b/crates/sqlbuild-config/src/command_connection/models.rs new file mode 100644 index 0000000000..5b89fcb88d --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/models.rs @@ -0,0 +1,28 @@ +//! A target's authored command-group warehouse. + +/// One target's authored, unexpanded warehouse for a command group and the file setting it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuthoredWarehouse { + pub warehouse: String, + pub target_name: String, + /// The `WarehouseGroup` value, `build` or `query`. + pub group: String, + pub file_name: String, +} + +impl AuthoredWarehouse { + /// ` [targets..warehouses] `. + #[must_use] + pub fn source(&self) -> String { + format!( + "{} [targets.{}.warehouses] {}", + self.file_name, self.target_name, self.group + ) + } + + /// The template label of the authored value. + #[must_use] + pub fn context_label(&self) -> String { + format!("targets.{}.warehouses.{}", self.target_name, self.group) + } +} diff --git a/crates/sqlbuild-config/src/command_connection/tests.rs b/crates/sqlbuild-config/src/command_connection/tests.rs new file mode 100644 index 0000000000..c811909c00 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_command_connection.rs"] +mod command_connection; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/command_connection/tests/helpers.rs b/crates/sqlbuild-config/src/command_connection/tests/helpers.rs new file mode 100644 index 0000000000..27bff2ba5d --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/tests/helpers.rs @@ -0,0 +1,5 @@ +use crate::command_connection::errors::CommandConnectionError; + +pub(super) fn message(result: Result<(), CommandConnectionError>) -> Option { + result.err().map(|error| error.message().to_owned()) +} diff --git a/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs b/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs new file mode 100644 index 0000000000..4c4c02071a --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs @@ -0,0 +1,137 @@ +use crate::command_connection::errors::CommandConnectionError; +use crate::command_connection::main::authored_warehouse::authored_warehouse; +use crate::command_connection::main::check_cli_warehouse::check_cli_warehouse; +use crate::command_connection::main::check_expanded_warehouse::check_expanded_warehouse; +use crate::command_connection::main::snowflake_key_warning::snowflake_key_warning; +use crate::command_connection::tests::helpers::message; +use crate::command_connection::tests::test_types::{ + CliWarehouseTestCase, ExpandedWarehouseTestCase, SnowflakeKeyTestCase, +}; + +#[test] +fn given_cli_warehouse_when_checking_then_matches_python_identifier_rules() { + let test_cases = [ + CliWarehouseTestCase { + description: "unquoted identifier with dollar", + cli_warehouse: "ADHOC_WH$1", + expected_message: None, + }, + CliWarehouseTestCase { + description: "quoted identifier with doubled quotes", + cli_warehouse: "\"Adhoc \"\" WH\"", + expected_message: None, + }, + CliWarehouseTestCase { + description: "blank value", + cli_warehouse: " ", + expected_message: Some("--warehouse requires a warehouse name"), + }, + CliWarehouseTestCase { + description: "lone quote inside a quoted identifier", + cli_warehouse: "\"a\"b\"", + expected_message: Some( + "--warehouse '\"a\"b\"' is not a valid Snowflake warehouse identifier", + ), + }, + CliWarehouseTestCase { + description: "empty quoted identifier", + cli_warehouse: "\"\"", + expected_message: Some( + "--warehouse '\"\"' is not a valid Snowflake warehouse identifier", + ), + }, + CliWarehouseTestCase { + description: "space in an unquoted identifier", + cli_warehouse: "bad wh", + expected_message: Some( + "--warehouse 'bad wh' is not a valid Snowflake warehouse identifier", + ), + }, + CliWarehouseTestCase { + description: "non-ASCII letter", + cli_warehouse: "wä", + expected_message: Some( + "--warehouse 'wä' is not a valid Snowflake warehouse identifier", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + message(check_cli_warehouse(test_case.cli_warehouse)).as_deref(), + test_case.expected_message, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_expanded_group_warehouse_when_checking_then_invalid_values_show_the_setting() { + let test_cases = [ + ExpandedWarehouseTestCase { + description: "project value that is not an identifier", + local_value: None, + project_value: Some("${wh}"), + expanded: "bad wh", + expected_result: Err(CommandConnectionError::User { + code: Some("C261"), + message: "sqlbuild_project.toml [targets.dev.warehouses] build resolves to 'bad wh', which is not a valid Snowflake warehouse identifier".to_owned(), + help: Some("to use an unquoted identifier (or a double-quoted one such as \"Adhoc WH\"), set this in sqlbuild_project.toml:\n [targets.dev.warehouses]\n build = \"ADHOC_WH\"".to_owned()), + }), + }, + ExpandedWarehouseTestCase { + description: "local value wins and is valid", + local_value: Some("LOCAL_WH"), + project_value: Some("bad wh"), + expanded: "LOCAL_WH", + expected_result: Ok(()), + }, + ]; + + for test_case in test_cases { + let result: Result<(), CommandConnectionError> = authored_warehouse( + ("dev", "build"), + test_case.local_value.map(str::to_owned), + test_case.project_value.map(str::to_owned), + ) + .map_or(Ok(()), |authored| { + check_expanded_warehouse(&authored, test_case.expanded) + }); + assert_eq!( + result, test_case.expected_result, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_connection_keys_when_warning_then_only_duckdb_snowflake_keys_are_listed() { + let test_cases = [ + SnowflakeKeyTestCase { + description: "sorted snowflake keys on duckdb", + adapter_name: "duckdb", + keys: &["warehouse", "database", "account"], + expected_warning: Some( + "Warning: DuckDB adapter is active, but connection contains Snowflake-like keys: account, warehouse. If this is a Snowflake local config, add `adapter = \"snowflake\"` at the top of sqlbuild_local.toml, before any section.", + ), + }, + SnowflakeKeyTestCase { + description: "snowflake adapter never warns", + adapter_name: "snowflake", + keys: &["account"], + expected_warning: None, + }, + ]; + + for test_case in test_cases { + let keys: Vec = test_case.keys.iter().map(|key| (*key).to_owned()).collect(); + assert_eq!( + snowflake_key_warning(test_case.adapter_name, &keys).as_deref(), + test_case.expected_warning, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/command_connection/tests/test_types.rs b/crates/sqlbuild-config/src/command_connection/tests/test_types.rs new file mode 100644 index 0000000000..bc3e6c8362 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/tests/test_types.rs @@ -0,0 +1,22 @@ +use crate::command_connection::errors::CommandConnectionError; + +pub(super) struct CliWarehouseTestCase { + pub(super) description: &'static str, + pub(super) cli_warehouse: &'static str, + pub(super) expected_message: Option<&'static str>, +} + +pub(super) struct SnowflakeKeyTestCase { + pub(super) description: &'static str, + pub(super) adapter_name: &'static str, + pub(super) keys: &'static [&'static str], + pub(super) expected_warning: Option<&'static str>, +} + +pub(super) struct ExpandedWarehouseTestCase { + pub(super) description: &'static str, + pub(super) local_value: Option<&'static str>, + pub(super) project_value: Option<&'static str>, + pub(super) expanded: &'static str, + pub(super) expected_result: Result<(), CommandConnectionError>, +} diff --git a/crates/sqlbuild-config/src/lib.rs b/crates/sqlbuild-config/src/lib.rs index 760d605ad1..d2ffd151bf 100644 --- a/crates/sqlbuild-config/src/lib.rs +++ b/crates/sqlbuild-config/src/lib.rs @@ -2,10 +2,12 @@ #![forbid(unsafe_code)] +pub mod command_connection; pub(crate) mod constants; pub mod effective; pub mod errors; pub mod models; pub mod project; pub mod toml; +pub mod workspace; pub mod yaml; diff --git a/crates/sqlbuild-config/src/workspace/_helpers/files.rs b/crates/sqlbuild-config/src/workspace/_helpers/files.rs new file mode 100644 index 0000000000..548380b90e --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/_helpers/files.rs @@ -0,0 +1,97 @@ +//! Text reads, atomic writes, template copies and directory walks. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::workspace::constants::{PYTHON_TEMPLATE_SUFFIX, TEMPLATE_SUFFIX}; +use crate::workspace::errors::WorkspaceError; + +static TEMPORARY_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// `path.read_text(encoding="utf-8")`. +pub(crate) fn read_text(path: &Path) -> Result { + let bytes: Vec = fs::read(path)?; + match String::from_utf8(bytes) { + Ok(text) => Ok(text), + Err(error) => { + let utf8: std::str::Utf8Error = error.utf8_error(); + Err(WorkspaceError::Decode { + path: path.to_path_buf(), + bytes: error.into_bytes(), + error: utf8, + }) + } + } +} + +/// Replace `path` with `content` through a temporary sibling file. +pub(crate) fn write_atomically(path: &Path, content: &str) -> Result<(), WorkspaceError> { + let parent: &Path = path.parent().unwrap_or_else(|| Path::new("")); + fs::create_dir_all(parent)?; + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let temporary: PathBuf = parent.join(format!( + ".{name}.{}.{}", + process::id(), + TEMPORARY_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + fs::write(&temporary, content)?; + fs::rename(&temporary, path)?; + Ok(()) +} + +/// The file name a template entry is written as: `.py.txt` loses `.txt`. +fn generated_name(name: &str) -> &str { + if name.ends_with(PYTHON_TEMPLATE_SUFFIX) { + return &name[..name.len() - TEMPLATE_SUFFIX.len()]; + } + name +} + +/// Copy the template entries of `source` into the existing directory `target`. +pub(crate) fn copy_entries(source: &Path, target: &Path) -> Result<(), WorkspaceError> { + for entry in fs::read_dir(source)? { + let entry: fs::DirEntry = entry?; + let name: String = entry.file_name().to_string_lossy().into_owned(); + let destination: PathBuf = target.join(generated_name(&name)); + if entry.path().is_dir() { + fs::create_dir(&destination)?; + copy_entries(&entry.path(), &destination)?; + } else { + fs::write(&destination, fs::read(entry.path())?)?; + } + } + Ok(()) +} + +/// `_copy_tree`: create `target` (and its parents) and copy the template into it. +pub(crate) fn copy_tree(source: &Path, target: &Path) -> Result<(), WorkspaceError> { + if target.exists() { + return Err(WorkspaceError::Io(std::io::Error::from( + std::io::ErrorKind::AlreadyExists, + ))); + } + fs::create_dir_all(target)?; + copy_entries(source, target) +} + +/// Every entry below `root`, without following directory symlinks. +pub(crate) fn walk(root: &Path) -> Vec { + let mut found: Vec = Vec::new(); + let Ok(entries) = fs::read_dir(root) else { + return found; + }; + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + let is_directory: bool = entry.file_type().is_ok_and(|kind| kind.is_dir()); + found.push(path.clone()); + if is_directory { + found.extend(walk(&path)); + } + } + found +} diff --git a/crates/sqlbuild-config/src/workspace/_helpers/mod.rs b/crates/sqlbuild-config/src/workspace/_helpers/mod.rs new file mode 100644 index 0000000000..1fc64506be --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod files; +pub(crate) mod playground; +pub(crate) mod skill_files; +pub(crate) mod skill_settings; diff --git a/crates/sqlbuild-config/src/workspace/_helpers/playground.rs b/crates/sqlbuild-config/src/workspace/_helpers/playground.rs new file mode 100644 index 0000000000..cf2d2d33d2 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/_helpers/playground.rs @@ -0,0 +1,80 @@ +//! Playground template choice and the commands its summary suggests. + +use std::path::{Path, PathBuf}; + +use crate::workspace::_helpers::files::{copy_entries, copy_tree}; +use crate::workspace::constants::{ + DAGSTER_COMMAND, DAGSTER_TEMPLATE, DEFAULT_EXAMPLE, LOADER_WAFFLE_SHOP_TEMPLATE, + ORCHESTRATED_PROJECT_DIR, ORCHESTRATED_TEMPLATES, PLAYGROUND_EXAMPLES, PROJECT_COMMANDS, + PYTHON_NODES_COMMANDS, PYTHON_NODES_TEMPLATE, RIVERS_COMMAND, RIVERS_TEMPLATE, + WAFFLE_SHOP_TEMPLATE, +}; +use crate::workspace::errors::WorkspaceError; + +/// Whether the template puts the project under `waffle_shop/` beside orchestrator files. +pub(crate) fn orchestrated(template: &str) -> bool { + ORCHESTRATED_TEMPLATES.contains(&template) +} + +/// Copy the chosen template into the new target directory. +pub(crate) fn write_template( + root: &Path, + target: &Path, + template: &str, +) -> Result<(), WorkspaceError> { + if template == PYTHON_NODES_TEMPLATE { + return copy_tree(&root.join(PYTHON_NODES_TEMPLATE), target); + } + let project: &str = if template == LOADER_WAFFLE_SHOP_TEMPLATE { + LOADER_WAFFLE_SHOP_TEMPLATE + } else { + WAFFLE_SHOP_TEMPLATE + }; + let project_root: PathBuf = root.join(project); + if !project_root.is_dir() { + return Err(WorkspaceError::user( + "C702", + "packaged playground template is missing".to_owned(), + Some("reinstall SQLBuild or report a packaging issue"), + )); + } + if !orchestrated(template) { + return copy_tree(&project_root, target); + } + copy_tree(&project_root, &target.join(ORCHESTRATED_PROJECT_DIR))?; + copy_entries(&root.join(template), target) +} + +/// `_suggested_commands`. +pub(crate) fn suggested_commands(template: &str, display_path: &str) -> Vec { + let project_path: String = if orchestrated(template) { + format!("{display_path}/{ORCHESTRATED_PROJECT_DIR}") + } else { + display_path.to_owned() + }; + let mut commands: Vec = vec![format!("cd {project_path}")]; + let steps: &[&str] = if template == PYTHON_NODES_TEMPLATE { + &PYTHON_NODES_COMMANDS + } else { + &PROJECT_COMMANDS + }; + commands.extend(steps.iter().map(|step| (*step).to_owned())); + if orchestrated(template) { + commands.push("cd ..".to_owned()); + } + if template == DAGSTER_TEMPLATE { + commands.push(DAGSTER_COMMAND.to_owned()); + } + if template == RIVERS_TEMPLATE { + commands.push(RIVERS_COMMAND.to_owned()); + } + commands +} + +/// `_example_name`. +pub(crate) fn example_name(template: &str) -> &'static str { + PLAYGROUND_EXAMPLES + .iter() + .find(|(name, _)| *name == template) + .map_or(DEFAULT_EXAMPLE, |(_, label)| label) +} diff --git a/crates/sqlbuild-config/src/workspace/_helpers/skill_files.rs b/crates/sqlbuild-config/src/workspace/_helpers/skill_files.rs new file mode 100644 index 0000000000..9f03941f22 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/_helpers/skill_files.rs @@ -0,0 +1,184 @@ +//! Packaged skill files and the generated copies installed from them. + +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::workspace::_helpers::files::{read_text, walk, write_atomically}; +use crate::workspace::constants::{ + FRONTMATTER_CLOSING, FRONTMATTER_OPENING, GENERATED_MARKER, LEGACY_GENERATED_MARKER, + SKILL_ENTRY_FILE, SKILL_FILE_SUFFIX, SKIPPED_SKILL_PREFIXES, +}; +use crate::workspace::errors::WorkspaceError; + +/// Skill-relative paths and their generated contents, sorted by path. +pub(crate) type SkillFiles = BTreeMap; + +/// `ensure_generated_marker`. +pub(crate) fn with_marker(content: String) -> String { + if content.contains(GENERATED_MARKER) { + return content; + } + if content.contains(LEGACY_GENERATED_MARKER) { + return content.replacen(LEGACY_GENERATED_MARKER, GENERATED_MARKER, 1); + } + if content.starts_with(FRONTMATTER_OPENING) + && let Some(position) = content[FRONTMATTER_OPENING.len()..].find(FRONTMATTER_CLOSING) + { + let insert_at: usize = FRONTMATTER_OPENING.len() + position + FRONTMATTER_CLOSING.len(); + return format!( + "{}\n{GENERATED_MARKER}\n{}", + &content[..insert_at], + &content[insert_at..] + ); + } + format!("{GENERATED_MARKER}\n{content}") +} + +fn collect(node: &Path, prefix: &str) -> Result { + let mut files: SkillFiles = SkillFiles::new(); + for entry in fs::read_dir(node)? { + let entry: fs::DirEntry = entry?; + let name: String = entry.file_name().to_string_lossy().into_owned(); + if SKIPPED_SKILL_PREFIXES + .iter() + .any(|skipped| name.starts_with(skipped)) + { + continue; + } + if entry.path().is_dir() { + files.extend(collect(&entry.path(), &format!("{prefix}{name}/"))?); + } else if name.ends_with(SKILL_FILE_SUFFIX) { + files.insert(format!("{prefix}{name}"), read_text(&entry.path())?); + } + } + Ok(files) +} + +/// `load_packaged_skill_files`: every packaged skill file with its generated marker. +pub(crate) fn packaged_files(root: &Path) -> Result { + if !root.join(SKILL_ENTRY_FILE).is_file() { + return Err(WorkspaceError::user( + "C805", + "packaged SQLBuild skill is missing".to_owned(), + Some("reinstall SQLBuild or report a packaging issue"), + )); + } + Ok(collect(root, "")? + .into_iter() + .map(|(path, content)| (path, with_marker(content))) + .collect()) +} + +fn generated(path: &Path) -> Result { + let content: String = read_text(path)?; + Ok(content.contains(GENERATED_MARKER) || content.contains(LEGACY_GENERATED_MARKER)) +} + +/// `_skill_collisions`: expected paths occupied by files SQLBuild did not generate. +pub(crate) fn collisions( + skill_dir: &Path, + files: &SkillFiles, +) -> Result, WorkspaceError> { + let mut found: Vec = Vec::new(); + for relative in files.keys() { + let path: PathBuf = skill_dir.join(relative); + if path.is_file() && !generated(&path)? { + found.push(path); + } + } + Ok(found) +} + +/// `_stale_generated_files`: generated `.md` files no longer shipped. +fn stale_generated(skill_dir: &Path, files: &SkillFiles) -> Result, WorkspaceError> { + if !skill_dir.is_dir() { + return Ok(Vec::new()); + } + let mut candidates: Vec = walk(skill_dir) + .into_iter() + .filter(|path| path.to_string_lossy().ends_with(SKILL_FILE_SUFFIX)) + .collect(); + candidates.sort(); + let mut stale: Vec = Vec::new(); + for path in candidates { + let relative: String = path + .strip_prefix(skill_dir) + .map(|relative| relative.to_string_lossy().into_owned()) + .unwrap_or_default(); + if !files.contains_key(&relative) && generated(&path)? { + stale.push(path); + } + } + Ok(stale) +} + +fn current(path: &Path, content: &str) -> Result { + Ok(path.is_file() && read_text(path)? == content) +} + +/// `_skill_directory_is_stale`. +pub(crate) fn out_of_date(skill_dir: &Path, files: &SkillFiles) -> Result { + for (relative, content) in files { + if !current(&skill_dir.join(relative), content)? { + return Ok(true); + } + } + Ok(!stale_generated(skill_dir, files)?.is_empty()) +} + +fn refuse(path: &Path) -> WorkspaceError { + WorkspaceError::user( + "C806", + format!( + "refusing to overwrite non-generated skill file: {}", + path.display() + ), + Some("rerun with --force to replace it"), + ) +} + +/// `_skill_collisions` over several install directories: the first collision refuses the install. +pub(crate) fn refuse_collisions( + skill_dirs: &[PathBuf], + files: &SkillFiles, +) -> Result<(), WorkspaceError> { + for skill_dir in skill_dirs { + if let Some(path) = collisions(skill_dir, files)?.into_iter().next() { + return Err(refuse(&path)); + } + } + Ok(()) +} + +/// `_install_skill_directory`: write every packaged file, drop unshipped ones and empty directories. +pub(crate) fn install( + skill_dir: &Path, + files: &SkillFiles, + force: bool, +) -> Result<(), WorkspaceError> { + for (relative, content) in files { + let path: PathBuf = skill_dir.join(relative); + if current(&path, content)? { + continue; + } + if path.exists() && !force && !generated(&path)? { + return Err(refuse(&path)); + } + write_atomically(&path, content)?; + } + for path in stale_generated(skill_dir, files)? { + fs::remove_file(path)?; + } + let mut directories: Vec = walk(skill_dir) + .into_iter() + .filter(|path| path.is_dir()) + .collect(); + directories.sort_by_key(|path| std::cmp::Reverse(path.components().count())); + for directory in directories { + if fs::read_dir(&directory)?.next().is_none() { + fs::remove_dir(&directory)?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs b/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs new file mode 100644 index 0000000000..4c0961fa12 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs @@ -0,0 +1,174 @@ +//! `[skills]` project settings and skill target names. + +use std::path::{Path, PathBuf}; + +use crate::models::ConfigValue; +use crate::toml::main::load_toml::load_toml; +use crate::workspace::_helpers::files::read_text; +use crate::workspace::constants::{ + CLAUDE_TARGET, DEFAULT_SKILL_TARGETS, GIT_MARKER, OPENCODE_TARGET, PROJECT_CONFIG_FILENAME, + SKILL_ENTRY_FILE, SKILL_NAME, UTF8_BYTE_ORDER_MARK, VALID_SKILL_TARGETS, +}; +use crate::workspace::errors::WorkspaceError; + +/// Project skill installation preferences (`SkillSettings`). +pub(crate) struct SkillSettings { + pub(crate) targets: Vec, + pub(crate) auto_update: bool, + pub(crate) configured: bool, +} + +fn unconfigured() -> SkillSettings { + SkillSettings { + targets: Vec::new(), + auto_update: false, + configured: false, + } +} + +/// The default targets, `agents` and `claude`. +pub(crate) fn default_targets() -> Vec { + DEFAULT_SKILL_TARGETS + .iter() + .map(|target| (*target).to_owned()) + .collect() +} + +/// `normalize_skill_targets`: lower-cased, validated, first occurrence kept. +pub(crate) fn normalize_targets( + targets: &[String], + source: &str, +) -> Result, WorkspaceError> { + let mut normalized: Vec = Vec::new(); + for target in targets { + let name: String = target.trim().to_lowercase(); + if !VALID_SKILL_TARGETS.contains(&name.as_str()) { + return Err(WorkspaceError::user( + "C801", + format!("{source} contains unsupported skill target '{target}'"), + Some(&format!( + "supported targets are: {}", + VALID_SKILL_TARGETS.join(", ") + )), + )); + } + if !normalized.contains(&name) { + normalized.push(name); + } + } + Ok(normalized) +} + +fn entry<'a>(map: &'a [(ConfigValue, ConfigValue)], key: &str) -> Option<&'a ConfigValue> { + map.iter() + .find(|(name, _)| matches!(name, ConfigValue::String(text) if text == key)) + .map(|(_, value)| value) +} + +fn string_list(value: &ConfigValue) -> Option> { + let ConfigValue::List(items) = value else { + return None; + }; + items + .iter() + .map(|item| match item { + ConfigValue::String(text) => Some(text.clone()), + _ => None, + }) + .collect() +} + +/// `load_project_skill_settings`. +pub(crate) fn load_settings(project_dir: &Path) -> Result { + let path: PathBuf = project_dir.join(PROJECT_CONFIG_FILENAME); + if !path.exists() { + return Ok(unconfigured()); + } + let text: String = read_text(&path)?; + let text: &str = text.strip_prefix(UTF8_BYTE_ORDER_MARK).unwrap_or(&text); + let shown: String = path.display().to_string(); + let document: ConfigValue = load_toml(text).map_err(|error| { + WorkspaceError::user( + "C802", + format!("{shown} contains invalid TOML: {}", error.message), + None, + ) + })?; + let ConfigValue::Map(document) = document else { + return Ok(unconfigured()); + }; + let Some(skills) = entry(&document, "skills") else { + return Ok(unconfigured()); + }; + let ConfigValue::Map(skills) = skills else { + return Err(WorkspaceError::user( + "C803", + format!("{shown} [skills] must be a table"), + None, + )); + }; + let targets: Option> = match entry(skills, "targets") { + None => None, + Some(value) => Some(string_list(value).ok_or_else(|| { + WorkspaceError::user( + "C804", + format!("{shown} skills.targets must be a list of strings"), + None, + ) + })?), + }; + let auto_update: bool = match entry(skills, "auto_update") { + None => false, + Some(ConfigValue::Bool(flag)) => *flag, + Some(_) => { + return Err(WorkspaceError::user( + "C808", + format!("{shown} skills.auto_update must be a boolean"), + None, + )); + } + }; + let targets: Vec = match targets { + Some(targets) => normalize_targets(&targets, "skills.targets")?, + None => Vec::new(), + }; + Ok(SkillSettings { + targets: if targets.is_empty() { + default_targets() + } else { + targets + }, + auto_update, + configured: true, + }) +} + +/// `_find_git_root`: the nearest directory, from `project_dir` up, holding `.git`. +fn git_root(project_dir: &Path) -> Option<&Path> { + project_dir + .ancestors() + .find(|candidate| candidate.join(GIT_MARKER).exists()) +} + +/// `build_install_targets`: the `SKILL.md` path of every target. +pub(crate) fn install_paths( + project_dir: &Path, + targets: &[String], + home: Option<&Path>, +) -> Vec { + let local: &Path = git_root(project_dir).unwrap_or(project_dir); + targets + .iter() + .map(|target| { + let base: PathBuf = match (target.as_str(), home) { + (OPENCODE_TARGET, Some(home)) => home.join(".config/opencode"), + (OPENCODE_TARGET, None) => local.join(".opencode"), + (CLAUDE_TARGET, Some(home)) => home.join(".claude"), + (CLAUDE_TARGET, None) => local.join(".claude"), + (_, Some(home)) => home.join(".agents"), + (_, None) => local.join(".agents"), + }; + base.join("skills").join(SKILL_NAME).join(SKILL_ENTRY_FILE) + }) + .collect() +} diff --git a/crates/sqlbuild-config/src/workspace/constants.rs b/crates/sqlbuild-config/src/workspace/constants.rs new file mode 100644 index 0000000000..811834e0bf --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/constants.rs @@ -0,0 +1,49 @@ +//! Playground templates, skill install locations and generated-file markers. + +pub(crate) const PLAYGROUND_TEMPLATES: [&str; 5] = [ + "waffle_shop", + "loader_waffle_shop", + "dagster", + "rivers", + "python_nodes", +]; +pub(crate) const WAFFLE_SHOP_TEMPLATE: &str = "waffle_shop"; +pub(crate) const LOADER_WAFFLE_SHOP_TEMPLATE: &str = "loader_waffle_shop"; +pub(crate) const PYTHON_NODES_TEMPLATE: &str = "python_nodes"; +pub(crate) const DAGSTER_TEMPLATE: &str = "dagster"; +pub(crate) const RIVERS_TEMPLATE: &str = "rivers"; +pub(crate) const ORCHESTRATED_TEMPLATES: [&str; 2] = [DAGSTER_TEMPLATE, RIVERS_TEMPLATE]; +pub(crate) const ORCHESTRATED_PROJECT_DIR: &str = "waffle_shop"; +pub(crate) const PYTHON_TEMPLATE_SUFFIX: &str = ".py.txt"; +pub(crate) const TEMPLATE_SUFFIX: &str = ".txt"; +pub(crate) const GENERATED_MARKER: &str = ""; +pub(crate) const LEGACY_GENERATED_MARKER: &str = ""; +pub(crate) const FRONTMATTER_OPENING: &str = "---\n"; +pub(crate) const FRONTMATTER_CLOSING: &str = "\n---\n"; +pub(crate) const SKILL_NAME: &str = "sqlbuild"; +pub(crate) const SKILL_ENTRY_FILE: &str = "SKILL.md"; +pub(crate) const SKILL_FILE_SUFFIX: &str = ".md"; +pub(crate) const SKIPPED_SKILL_PREFIXES: [&str; 2] = [".", "__"]; +pub(crate) const VALID_SKILL_TARGETS: [&str; 3] = ["opencode", "claude", "agents"]; +pub(crate) const DEFAULT_SKILL_TARGETS: [&str; 2] = ["agents", "claude"]; +pub(crate) const OPENCODE_TARGET: &str = "opencode"; +pub(crate) const CLAUDE_TARGET: &str = "claude"; +pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +pub(crate) const UTF8_BYTE_ORDER_MARK: char = '\u{feff}'; +pub(crate) const GIT_MARKER: &str = ".git"; +pub(crate) const PLAYGROUND_EXAMPLES: [(&str, &str); 4] = [ + (DAGSTER_TEMPLATE, "waffle shop + Dagster"), + (RIVERS_TEMPLATE, "waffle shop + Rivers"), + (LOADER_WAFFLE_SHOP_TEMPLATE, "loader-focused waffle shop"), + (PYTHON_NODES_TEMPLATE, "Python nodes demo"), +]; +pub(crate) const DEFAULT_EXAMPLE: &str = "waffle shop"; +pub(crate) const PYTHON_NODES_COMMANDS: [&str; 3] = [ + "sqb plan --select +fact_orders --select +orders_export", + "sqb build --select +fact_orders --select +orders_export", + "sqb check --select check_orders_export", +]; +pub(crate) const PROJECT_COMMANDS: [&str; 4] = + ["sqb compile", "sqb build", "sqb test", "sqb audit"]; +pub(crate) const DAGSTER_COMMAND: &str = "DAGSTER_IS_DEV_CLI=1 dagster dev -f definitions.py"; +pub(crate) const RIVERS_COMMAND: &str = "rivers dev definitions"; diff --git a/crates/sqlbuild-config/src/workspace/errors.rs b/crates/sqlbuild-config/src/workspace/errors.rs new file mode 100644 index 0000000000..a9084268cd --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/errors.rs @@ -0,0 +1,39 @@ +//! Why a playground or skill install could not be written. + +use std::io; +use std::path::PathBuf; +use std::str::Utf8Error; + +/// A coded user error, a file that is not UTF-8 text, or an I/O failure. +#[derive(Debug)] +pub enum WorkspaceError { + /// `CliUserError(message, code=code, help=help)`. + User { + code: &'static str, + message: String, + help: Option, + }, + /// A text file that does not decode as UTF-8. + Decode { + path: PathBuf, + bytes: Vec, + error: Utf8Error, + }, + Io(io::Error), +} + +impl WorkspaceError { + pub(crate) fn user(code: &'static str, message: String, help: Option<&str>) -> Self { + Self::User { + code, + message, + help: help.map(str::to_owned), + } + } +} + +impl From for WorkspaceError { + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} diff --git a/crates/sqlbuild-config/src/workspace/main/create_playground.rs b/crates/sqlbuild-config/src/workspace/main/create_playground.rs new file mode 100644 index 0000000000..95c09dcd1a --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/main/create_playground.rs @@ -0,0 +1,35 @@ +//! Create a playground project from the packaged templates. + +use crate::workspace::_helpers::playground::write_template; +use crate::workspace::constants::PLAYGROUND_TEMPLATES; +use crate::workspace::errors::WorkspaceError; +use crate::workspace::models::PlaygroundRequest; + +/// `create_playground_project`: refuse an existing target or unknown template, then copy. +pub fn create_playground(request: &PlaygroundRequest) -> Result<(), WorkspaceError> { + if request.target_dir.exists() { + return Err(WorkspaceError::user( + "C701", + format!( + "playground target already exists: {}", + request.target_dir.display() + ), + Some("choose a new directory or remove the existing one"), + )); + } + if !PLAYGROUND_TEMPLATES.contains(&request.template.as_str()) { + return Err(WorkspaceError::user( + "C703", + format!("unknown playground template: {}", request.template), + Some(&format!( + "choose one of: {}", + PLAYGROUND_TEMPLATES.join(", ") + )), + )); + } + write_template( + &request.templates_root, + &request.target_dir, + &request.template, + ) +} diff --git a/crates/sqlbuild-config/src/workspace/main/install_skills.rs b/crates/sqlbuild-config/src/workspace/main/install_skills.rs new file mode 100644 index 0000000000..400b7046d8 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/main/install_skills.rs @@ -0,0 +1,42 @@ +//! `sqb skills`: install the packaged SQLBuild skill for each agent target. + +use std::path::{Path, PathBuf}; + +use crate::workspace::_helpers::skill_files::{ + SkillFiles, install, packaged_files, refuse_collisions, +}; +use crate::workspace::_helpers::skill_settings::{ + SkillSettings, default_targets, install_paths, load_settings, normalize_targets, +}; +use crate::workspace::errors::WorkspaceError; +use crate::workspace::models::SkillInstallRequest; + +fn target_names(request: &SkillInstallRequest) -> Result, WorkspaceError> { + if !request.requested_targets.is_empty() { + return normalize_targets(&request.requested_targets, "--target"); + } + let settings: SkillSettings = load_settings(&request.project_dir)?; + if settings.targets.is_empty() { + return Ok(default_targets()); + } + Ok(settings.targets) +} + +/// `update_sqlbuild_skills`: the `SKILL.md` path written for every target. +pub fn install_skills(request: &SkillInstallRequest) -> Result, WorkspaceError> { + let targets: Vec = target_names(request)?; + let home: Option<&Path> = request.global_install.then_some(request.home_dir.as_path()); + let paths: Vec = install_paths(&request.project_dir, &targets, home); + let files: SkillFiles = packaged_files(&request.skills_root)?; + let skill_dirs: Vec = paths + .iter() + .map(|path| path.parent().map(Path::to_path_buf).unwrap_or_default()) + .collect(); + if !request.force { + refuse_collisions(&skill_dirs, &files)?; + } + for skill_dir in &skill_dirs { + install(skill_dir, &files, request.force)?; + } + Ok(paths) +} diff --git a/crates/sqlbuild-config/src/workspace/main/maintain_skills.rs b/crates/sqlbuild-config/src/workspace/main/maintain_skills.rs new file mode 100644 index 0000000000..ce5f003173 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/main/maintain_skills.rs @@ -0,0 +1,76 @@ +//! Check or refresh a project's generated skills without changing a command's outcome. + +use std::path::{Path, PathBuf}; + +use crate::workspace::_helpers::skill_files::{ + SkillFiles, collisions, install, out_of_date, packaged_files, +}; +use crate::workspace::_helpers::skill_settings::{SkillSettings, install_paths, load_settings}; +use crate::workspace::constants::VALID_SKILL_TARGETS; +use crate::workspace::errors::WorkspaceError; + +/// `_stale_skill_message`. +fn stale_message(collision_paths: &[PathBuf]) -> String { + let mut detail: String = String::new(); + if !collision_paths.is_empty() { + detail.push_str(" Custom files were not overwritten:\n"); + for path in collision_paths { + detail.push_str(&format!(" {}\n", path.display())); + } + } + format!("SQLBuild skill files are out of date\n{detail} Run: sqb skills\n") +} + +fn installed_targets(project_dir: &Path) -> Vec { + VALID_SKILL_TARGETS + .iter() + .map(|target| (*target).to_owned()) + .filter(|target| { + install_paths(project_dir, std::slice::from_ref(target), None) + .first() + .is_some_and(|path| path.exists()) + }) + .collect() +} + +/// `maintain_sqlbuild_skills`: the notice to show, empty when skills are current. +pub fn maintain_skills(project_dir: &Path, skills_root: &Path) -> Result { + let settings: SkillSettings = load_settings(project_dir)?; + let targets: Vec = if settings.configured { + settings.targets + } else { + installed_targets(project_dir) + }; + if targets.is_empty() { + return Ok(String::new()); + } + let files: SkillFiles = packaged_files(skills_root)?; + let mut stale: Vec = Vec::new(); + let mut collided: Vec = Vec::new(); + for path in install_paths(project_dir, &targets, None) { + let skill_dir: PathBuf = path.parent().map(Path::to_path_buf).unwrap_or_default(); + let found: Vec = collisions(&skill_dir, &files)?; + if !found.is_empty() { + collided.extend(found); + continue; + } + if out_of_date(&skill_dir, &files)? { + stale.push(path); + } + } + if settings.auto_update && !stale.is_empty() { + let mut message: String = "Updated stale SQLBuild skill files:\n".to_owned(); + for path in &stale { + install(path.parent().unwrap_or(path), &files, false)?; + message.push_str(&format!(" {}\n", path.display())); + } + if !collided.is_empty() { + message.push_str(&stale_message(&collided)); + } + return Ok(message); + } + if !stale.is_empty() || !collided.is_empty() { + return Ok(stale_message(&collided)); + } + Ok(String::new()) +} diff --git a/crates/sqlbuild-config/src/workspace/main/mod.rs b/crates/sqlbuild-config/src/workspace/main/mod.rs new file mode 100644 index 0000000000..5a5a093058 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/main/mod.rs @@ -0,0 +1,6 @@ +pub mod create_playground; +pub mod install_skills; +pub mod maintain_skills; +pub mod packaged_skill_files; +pub mod playground_project_dir; +pub mod playground_summary; diff --git a/crates/sqlbuild-config/src/workspace/main/packaged_skill_files.rs b/crates/sqlbuild-config/src/workspace/main/packaged_skill_files.rs new file mode 100644 index 0000000000..be96a2a75e --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/main/packaged_skill_files.rs @@ -0,0 +1,14 @@ +//! The packaged skill files as they are installed. + +use std::collections::BTreeMap; +use std::path::Path; + +use crate::workspace::_helpers::skill_files::packaged_files; +use crate::workspace::errors::WorkspaceError; + +/// Every packaged skill file by skill-relative path, with its generated marker. +pub fn packaged_skill_files( + skills_root: &Path, +) -> Result, WorkspaceError> { + packaged_files(skills_root) +} diff --git a/crates/sqlbuild-config/src/workspace/main/playground_project_dir.rs b/crates/sqlbuild-config/src/workspace/main/playground_project_dir.rs new file mode 100644 index 0000000000..d93eff3ed9 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/main/playground_project_dir.rs @@ -0,0 +1,15 @@ +//! The SQLBuild project directory inside a generated playground. + +use std::path::{Path, PathBuf}; + +use crate::workspace::_helpers::playground::orchestrated; +use crate::workspace::constants::ORCHESTRATED_PROJECT_DIR; + +/// `waffle_shop/` under orchestrated playgrounds, else the playground itself. +#[must_use] +pub fn playground_project_dir(target_dir: &Path, template: &str) -> PathBuf { + if orchestrated(template) { + return target_dir.join(ORCHESTRATED_PROJECT_DIR); + } + target_dir.to_path_buf() +} diff --git a/crates/sqlbuild-config/src/workspace/main/playground_summary.rs b/crates/sqlbuild-config/src/workspace/main/playground_summary.rs new file mode 100644 index 0000000000..8346f08030 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/main/playground_summary.rs @@ -0,0 +1,20 @@ +//! The summary printed after a playground is created. + +use sqlbuild_core::cli_text::models::{CliDocument, CliStyle}; + +use crate::workspace::_helpers::playground::{example_name, suggested_commands}; + +/// `render_playground_completion_text`. +#[must_use] +pub fn playground_summary(display_path: &str, template: &str, style: CliStyle) -> String { + let mut document: CliDocument = CliDocument::new(style); + document.header("SQLBuild playground created", None); + document.lines.push(String::new()); + document.field("Project", (display_path, " ")); + document.field("Adapter", ("DuckDB", " ")); + document.field("Example", (example_name(template), " ")); + document.lines.push(String::new()); + document.title_section("Try"); + document.commands(&suggested_commands(template, display_path), (false, false)); + document.render(true) +} diff --git a/crates/sqlbuild-config/src/workspace/mod.rs b/crates/sqlbuild-config/src/workspace/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/workspace/models.rs b/crates/sqlbuild-config/src/workspace/models.rs new file mode 100644 index 0000000000..5de142b3d5 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/models.rs @@ -0,0 +1,24 @@ +//! Requests to create a playground or install skills, and maintenance outcomes. + +use std::path::PathBuf; + +/// Where `sqb skills` installs, from where, and how. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SkillInstallRequest { + pub project_dir: PathBuf, + /// The packaged `.agents/skills/sqlbuild` directory. + pub skills_root: PathBuf, + pub home_dir: PathBuf, + pub global_install: bool, + pub requested_targets: Vec, + pub force: bool, +} + +/// One playground to create from the packaged templates. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PlaygroundRequest { + /// The packaged `playground_templates` directory. + pub templates_root: PathBuf, + pub target_dir: PathBuf, + pub template: String, +} diff --git a/crates/sqlbuild-config/src/workspace/tests.rs b/crates/sqlbuild-config/src/workspace/tests.rs new file mode 100644 index 0000000000..5b8139a1b9 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_workspace.rs"] +mod workspace; diff --git a/crates/sqlbuild-config/src/workspace/tests/test_types.rs b/crates/sqlbuild-config/src/workspace/tests/test_types.rs new file mode 100644 index 0000000000..2ffe60119c --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/tests/test_types.rs @@ -0,0 +1,11 @@ +pub(super) struct MarkerTestCase { + pub(super) description: &'static str, + pub(super) content: &'static str, + pub(super) expected_content: &'static str, +} + +pub(super) struct SummaryTestCase { + pub(super) description: &'static str, + pub(super) template: &'static str, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-config/src/workspace/tests/test_workspace.rs b/crates/sqlbuild-config/src/workspace/tests/test_workspace.rs new file mode 100644 index 0000000000..5753efeb55 --- /dev/null +++ b/crates/sqlbuild-config/src/workspace/tests/test_workspace.rs @@ -0,0 +1,53 @@ +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::workspace::_helpers::skill_files::with_marker; +use crate::workspace::main::playground_summary::playground_summary; +use crate::workspace::tests::test_types::{MarkerTestCase, SummaryTestCase}; + +#[test] +fn given_skill_content_when_marking_then_marker_follows_frontmatter() { + let test_cases = [ + MarkerTestCase { + description: "the marker follows frontmatter", + content: "---\nname: sqlbuild\n---\n# SQLBuild Skill\n", + expected_content: "---\nname: sqlbuild\n---\n\n\n# SQLBuild Skill\n", + }, + MarkerTestCase { + description: "the legacy marker is replaced", + content: "\nbody\n", + expected_content: "\nbody\n", + }, + MarkerTestCase { + description: "content without frontmatter starts with the marker", + content: "body\n", + expected_content: "\nbody\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + with_marker(test_case.content.to_owned()), + test_case.expected_content, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_playground_template_when_summarizing_then_next_steps_match_the_template() { + let test_cases = [SummaryTestCase { + description: "an orchestrated template enters and leaves the nested project", + template: "dagster", + expected_text: "SQLBuild playground created\n\n Project: demo\n Adapter: DuckDB\n Example: waffle shop + Dagster\n\nTry:\n cd demo/waffle_shop\n sqb compile\n sqb build\n sqb test\n sqb audit\n cd ..\n DAGSTER_IS_DEV_CLI=1 dagster dev -f definitions.py\n", + }]; + + for test_case in test_cases { + assert_eq!( + playground_summary("demo", test_case.template, CliStyle { use_color: false }), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs index 2fdfbd3544..f450e37ec1 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs @@ -3,4 +3,5 @@ pub(crate) mod diagnostics; pub(crate) mod layout; pub(crate) mod plan_document; pub(crate) mod plan_sections; +pub(crate) mod summary_text; pub(crate) mod tree_rows; diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/summary_text.rs b/crates/sqlbuild-core/src/cli_text/_helpers/summary_text.rs new file mode 100644 index 0000000000..df62efcb0a --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/summary_text.rs @@ -0,0 +1,116 @@ +//! Summary footers, completion lines and small aligned text pieces. + +use crate::cli_text::constants::{ + COUNT_HEADER_CLOSING, COUNT_HEADER_OPENING, FOOTER_ERROR_LABELS, FOOTER_SKIP_LABELS, + FOOTER_SUCCESS_LABELS, FOOTER_WARNING_LABELS, PHASE_FAIL_GLYPH, PHASE_OK_GLYPH, +}; +use crate::cli_text::models::{AlignedRow, CliStyle, CompletionState, SummaryCount}; + +fn footer_value(style: CliStyle, count: &SummaryCount) -> String { + let normalized: String = count.label.to_uppercase(); + let value: String = count.value.to_string(); + let label: &str = normalized.as_str(); + if FOOTER_SUCCESS_LABELS.contains(&label) { + return style.success(&value); + } + if FOOTER_WARNING_LABELS.contains(&label) { + return style.warning(&value); + } + if FOOTER_ERROR_LABELS.contains(&label) { + return style.error(&value); + } + if FOOTER_SKIP_LABELS.contains(&label) { + return style.muted(&value); + } + style.value(&value) +} + +/// `format_summary_footer`: `LABEL=value` pairs coloured by outcome, then the elapsed time. +pub(crate) fn summary_footer( + style: CliStyle, + counts: &[SummaryCount], + elapsed: Option<&str>, +) -> String { + let parts: Vec = counts + .iter() + .map(|count| { + format!( + "{}{}", + style.label(&format!("{}=", count.label)), + footer_value(style, count) + ) + }) + .collect(); + let rendered: String = parts.join(" "); + match elapsed { + Some(elapsed) => format!("{rendered} {}", style.muted(&format!("({elapsed})"))), + None => rendered, + } +} + +/// `format_count_noun`: the count with the singular noun for one, else the plural. +pub(crate) fn count_noun(count: i64, singular: &str, plural: Option<&str>) -> String { + if count == 1 { + return format!("{count} {singular}"); + } + match plural.filter(|plural| !plural.is_empty()) { + Some(plural) => format!("{count} {plural}"), + None => format!("{count} {singular}s"), + } +} + +/// `format_completion_line`: the state glyph, the label and the trailing summary. +pub(crate) fn completion_line( + style: CliStyle, + state: CompletionState, + label: &str, + summary: Option<&str>, +) -> String { + let (glyph, label): (String, String) = match state { + CompletionState::Fail => (style.error(PHASE_FAIL_GLYPH), style.error_strong(label)), + CompletionState::Warn => (style.warning(PHASE_OK_GLYPH), style.warning_strong(label)), + CompletionState::Ok => (style.success(PHASE_OK_GLYPH), label.to_owned()), + }; + match summary.filter(|summary| !summary.is_empty()) { + Some(summary) => format!("{glyph} {label} {summary}"), + None => format!("{glyph} {label}"), + } +} + +/// `format_aligned_name_value`: the row padded on the plain name's width. +pub(crate) fn aligned_row(row: &AlignedRow) -> String { + let padding: String = " ".repeat( + row.name_column_width + .saturating_sub(row.plain_name.chars().count()), + ); + format!("{}{}{padding} {}", row.prefix, row.styled_name, row.value) +} + +/// The title and count of a `Title (count)` header, when it has that shape. +pub(crate) fn count_header_parts(text: &str) -> Option<(&str, &str)> { + if !text.ends_with(COUNT_HEADER_CLOSING) { + return None; + } + let position: usize = text.rfind(COUNT_HEADER_OPENING)?; + Some(( + &text[..position], + &text[position + COUNT_HEADER_OPENING.len()..], + )) +} + +/// `resolve_name_column_width`: the longest name, at least `min_width`. +pub(crate) fn name_column_width(names: &[String], min_width: usize) -> usize { + names + .iter() + .map(|name| name.chars().count()) + .fold(min_width, usize::max) +} + +/// The overflow hint of a truncated section, when entries are hidden. +pub(crate) fn overflow_line(hidden: i64, indent: &str, flag: &str) -> Option { + (hidden > 0).then(|| { + CliStyle { use_color: true }.muted(&format!( + "{indent}... and {hidden} more (use {flag} to show all)" + )) + }) +} diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index 86d35be7a9..eae5ddd0a4 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -77,3 +77,10 @@ pub(crate) const READ_SIDE_PHASE: &str = "read_side"; pub(crate) const PYTHON_INGRESS_TITLE: &str = "Python ingress"; pub(crate) const PYTHON_READ_SIDE_TITLE: &str = "Python read-side"; pub(crate) const PLAN_READY_TITLE: &str = "Plan ready"; +pub(crate) const FOOTER_SUCCESS_LABELS: [&str; 5] = + ["PASS", "OK", "SYNC_PASS", "REFRESH_PASS", "UNCHANGED"]; +pub(crate) const FOOTER_WARNING_LABELS: [&str; 3] = ["WARN", "TOLERATED", "UNKNOWN"]; +pub(crate) const FOOTER_ERROR_LABELS: [&str; 4] = ["FAIL", "ERROR", "SYNC_FAIL", "REFRESH_FAIL"]; +pub(crate) const FOOTER_SKIP_LABELS: [&str; 1] = ["SKIP"]; +pub(crate) const COUNT_HEADER_OPENING: &str = " ("; +pub(crate) const COUNT_HEADER_CLOSING: &str = ")"; diff --git a/crates/sqlbuild-core/src/cli_text/main/format_aligned_name_value.rs b/crates/sqlbuild-core/src/cli_text/main/format_aligned_name_value.rs new file mode 100644 index 0000000000..f2ca73fdb3 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_aligned_name_value.rs @@ -0,0 +1,10 @@ +//! One aligned name/value row. + +use crate::cli_text::_helpers::summary_text::aligned_row; +use crate::cli_text::models::AlignedRow; + +/// The prefix, the styled name padded on the plain name's width, and the value. +#[must_use] +pub fn format_aligned_name_value(row: &AlignedRow) -> String { + aligned_row(row) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_completion_line.rs b/crates/sqlbuild-core/src/cli_text/main/format_completion_line.rs new file mode 100644 index 0000000000..7d5654d930 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_completion_line.rs @@ -0,0 +1,15 @@ +//! A single-line completion summary. + +use crate::cli_text::_helpers::summary_text::completion_line; +use crate::cli_text::models::{CliStyle, CompletionState}; + +/// The state glyph, the label and the trailing summary. +#[must_use] +pub fn format_completion_line( + style: CliStyle, + state: CompletionState, + label: &str, + summary: Option<&str>, +) -> String { + completion_line(style, state, label, summary) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_count_noun.rs b/crates/sqlbuild-core/src/cli_text/main/format_count_noun.rs new file mode 100644 index 0000000000..e0e3658349 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_count_noun.rs @@ -0,0 +1,9 @@ +//! A count with its noun. + +use crate::cli_text::_helpers::summary_text::count_noun; + +/// The count with the singular noun for one and the plural (default `s`) otherwise. +#[must_use] +pub fn format_count_noun(count: i64, singular: &str, plural: Option<&str>) -> String { + count_noun(count, singular, plural) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_overflow_line.rs b/crates/sqlbuild-core/src/cli_text/main/format_overflow_line.rs new file mode 100644 index 0000000000..39ac7984df --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_overflow_line.rs @@ -0,0 +1,9 @@ +//! The hint that a section shows only some of its entries. + +use crate::cli_text::_helpers::summary_text::overflow_line; + +/// `... and N more (use to show all)`, dim, when `hidden` entries are not shown. +#[must_use] +pub fn format_overflow_line(hidden: i64, indent: &str, flag: &str) -> Option { + overflow_line(hidden, indent, flag) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_phase_line.rs b/crates/sqlbuild-core/src/cli_text/main/format_phase_line.rs new file mode 100644 index 0000000000..1c1bf20509 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_phase_line.rs @@ -0,0 +1,10 @@ +//! The concise phase-completion line of human CLI output. + +use crate::cli_text::_helpers::layout::phase_line; +use crate::cli_text::models::CliStyle; + +/// `format_phase_line`: state glyph, label, dim summary. +#[must_use] +pub fn format_phase_line(style: CliStyle, ok: bool, (label, summary): (&str, &str)) -> String { + phase_line(style, ok, label, summary) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_status_cell.rs b/crates/sqlbuild-core/src/cli_text/main/format_status_cell.rs new file mode 100644 index 0000000000..30c41edc76 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_status_cell.rs @@ -0,0 +1,10 @@ +//! A fixed-width status cell. + +use crate::cli_text::_helpers::layout::status_cell; +use crate::cli_text::models::CliStyle; + +/// The styled status padded on its plain width. +#[must_use] +pub fn format_status_cell(style: CliStyle, status: &str, width: usize) -> String { + status_cell(style, status, width) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_summary_footer.rs b/crates/sqlbuild-core/src/cli_text/main/format_summary_footer.rs new file mode 100644 index 0000000000..7320d57e73 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_summary_footer.rs @@ -0,0 +1,14 @@ +//! The summary footer of a command's counts. + +use crate::cli_text::_helpers::summary_text::summary_footer; +use crate::cli_text::models::{CliStyle, SummaryCount}; + +/// `LABEL=value` pairs coloured by outcome, then the elapsed time when given. +#[must_use] +pub fn format_summary_footer( + style: CliStyle, + counts: &[SummaryCount], + elapsed: Option<&str>, +) -> String { + summary_footer(style, counts, elapsed) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_surface_header.rs b/crates/sqlbuild-core/src/cli_text/main/format_surface_header.rs new file mode 100644 index 0000000000..b2f3acbd3e --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_surface_header.rs @@ -0,0 +1,10 @@ +//! The command surface header of human CLI output. + +use crate::cli_text::_helpers::layout::surface_header; +use crate::cli_text::models::CliStyle; + +/// `format_surface_header`: accent title plus dim context. +#[must_use] +pub fn format_surface_header(style: CliStyle, title: &str, context: &str) -> String { + surface_header(style, title, context) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/mod.rs b/crates/sqlbuild-core/src/cli_text/main/mod.rs index 1d6c262c16..f64b0962c9 100644 --- a/crates/sqlbuild-core/src/cli_text/main/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/main/mod.rs @@ -1,4 +1,14 @@ +pub mod format_aligned_name_value; pub mod format_compile_text; +pub mod format_completion_line; +pub mod format_count_noun; +pub mod format_overflow_line; +pub mod format_phase_line; pub mod format_plan_completion; pub mod format_plan_text; +pub mod format_status_cell; +pub mod format_summary_footer; +pub mod format_surface_header; +pub mod resolve_name_column_width; pub mod semantic_coverage_notice; +pub mod split_count_header; diff --git a/crates/sqlbuild-core/src/cli_text/main/resolve_name_column_width.rs b/crates/sqlbuild-core/src/cli_text/main/resolve_name_column_width.rs new file mode 100644 index 0000000000..123edc1bb1 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/resolve_name_column_width.rs @@ -0,0 +1,9 @@ +//! The left column width of aligned name/value rows. + +use crate::cli_text::_helpers::summary_text::name_column_width; + +/// The longest name's width, at least `min_width`. +#[must_use] +pub fn resolve_name_column_width(names: &[String], min_width: usize) -> usize { + name_column_width(names, min_width) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/split_count_header.rs b/crates/sqlbuild-core/src/cli_text/main/split_count_header.rs new file mode 100644 index 0000000000..d26eb94fcb --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/split_count_header.rs @@ -0,0 +1,9 @@ +//! The parts of a `Title (count)` header. + +use crate::cli_text::_helpers::summary_text::count_header_parts; + +/// The title and the `count)` text, when the header ends with a parenthesized count. +#[must_use] +pub fn split_count_header(text: &str) -> Option<(&str, &str)> { + count_header_parts(text) +} diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 97843c1373..ac0d0a03b0 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -35,6 +35,21 @@ impl CliStyle { self.apply(DIM, text) } + #[must_use] + pub fn section(self, text: &str) -> String { + self.apply(BOLD, text) + } + + #[must_use] + pub fn command(self, text: &str) -> String { + self.apply(DIM, text) + } + + #[must_use] + pub fn value(self, text: &str) -> String { + self.apply(BLUE, text) + } + #[must_use] pub fn accent(self, text: &str) -> String { self.apply(BLUE, text) @@ -60,6 +75,11 @@ impl CliStyle { self.apply(GREEN, text) } + #[must_use] + pub fn success_strong(self, text: &str) -> String { + self.apply(&format!("{GREEN}{BOLD}"), text) + } + #[must_use] pub fn warning(self, text: &str) -> String { self.apply(YELLOW, text) @@ -70,6 +90,11 @@ impl CliStyle { self.apply(RED, text) } + #[must_use] + pub fn error_muted(self, text: &str) -> String { + self.apply(&format!("{RED}{DIM}"), text) + } + #[must_use] pub fn error_strong(self, text: &str) -> String { self.apply(&format!("{RED}{BOLD}"), text) @@ -78,20 +103,26 @@ impl CliStyle { /// `CliStyle.status`: the status word styled by what it reports. #[must_use] pub fn status(self, status: &str) -> String { + self.status_text(status, status) + } + + /// `CliStyle.status(status=..., text=...)`: `text` styled by the status word it reports. + #[must_use] + pub fn status_text(self, status: &str, text: &str) -> String { let normalized: String = status.to_uppercase(); if SUCCESS_STATUS_LABELS.contains(&normalized.as_str()) { - return self.success(status); + return self.success(text); } if WARNING_STATUS_LABELS.contains(&normalized.as_str()) { - return self.warning(status); + return self.warning(text); } if ERROR_STATUS_LABELS.contains(&normalized.as_str()) { - return self.error(status); + return self.error(text); } if SKIPPED_STATUS_LABELS.contains(&normalized.as_str()) { - return self.muted(status); + return self.muted(text); } - status.to_owned() + text.to_owned() } } @@ -362,3 +393,116 @@ pub(crate) enum RowKind { Nested, Other, } + +/// Overall outcome of a completion summary line (`CompletionState`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CompletionState { + Ok, + Warn, + Fail, +} + +/// One summary count: its label and value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SummaryCount { + pub label: String, + pub value: i64, +} + +/// One aligned name/value row: the plain name sets the width, the styled name is shown. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlignedRow { + pub plain_name: String, + pub styled_name: String, + pub value: String, + pub name_column_width: usize, + pub prefix: String, +} + +/// Simple static CLI output built line by line with semantic styling (`CliDocument`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CliDocument { + pub style: CliStyle, + pub lines: Vec, +} + +impl CliDocument { + #[must_use] + pub fn new(style: CliStyle) -> Self { + Self { + style, + lines: Vec::new(), + } + } + + pub fn header(&mut self, text: &str, suffix: Option<&str>) { + let title: String = self.style.title(text); + self.lines.push(match suffix { + Some(suffix) => format!("{title} {suffix}"), + None => title, + }); + } + + pub fn field(&mut self, label: &str, (value, padding): (&str, &str)) { + self.lines + .push(format!(" {}:{padding}{value}", self.style.value(label))); + } + + /// Rows of `label value`, labels left-justified to `label_width` when given. + pub fn fields(&mut self, rows: &[(String, String)], label_width: Option) { + for (label, value) in rows { + let label: String = match label_width { + Some(width) => format!("{label: label.clone(), + }; + self.lines + .push(format!(" {} {value}", self.style.value(&label))); + } + } + + pub fn section(&mut self, text: &str) { + self.lines.push(format!("{}:", self.style.section(text))); + } + + pub fn title_section(&mut self, text: &str) { + self.lines.push(format!("{}:", self.style.title(text))); + } + + pub fn items(&mut self, values: &[String]) { + for value in values { + self.lines + .push(format!(" {}", self.style.object_name(value))); + } + } + + pub fn command_line(&mut self, prefix: &str, (command, styled): (&str, bool)) { + let command: String = if styled { + self.style.command(command) + } else { + command.to_owned() + }; + self.lines.push(format!("{prefix}{command}")); + } + + /// Commands, numbered ` 1. ` or under a dim indent. + pub fn commands(&mut self, commands: &[String], (numbered, styled): (bool, bool)) { + for (index, command) in commands.iter().enumerate() { + let prefix: String = if numbered { + format!(" {}. ", index + 1) + } else { + self.style.command(" ") + }; + self.command_line(&prefix, (command, styled)); + } + } + + /// The lines joined, with a trailing newline when asked. + #[must_use] + pub fn render(&self, trailing_newline: bool) -> String { + let rendered: String = self.lines.join("\n"); + if trailing_newline { + return rendered + "\n"; + } + rendered + } +} diff --git a/crates/sqlbuild-core/src/cost_report/_helpers/breakdown.rs b/crates/sqlbuild-core/src/cost_report/_helpers/breakdown.rs new file mode 100644 index 0000000000..b38d65ee71 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/_helpers/breakdown.rs @@ -0,0 +1,66 @@ +//! The status, rate and note lines around a run's cost table. + +use crate::cli_text::models::CliStyle; +use crate::cost_report::_helpers::decimal_text::fixed; +use crate::cost_report::constants::{ + COLLECTION_FAILED_STATUS, COMPLETE_STATUS, DEFAULT_RATE_SOURCE, ESTIMATE_NOTE, RATE_ADVICE, + RATE_PLACES, WARNING_STATUSES, +}; +use crate::cost_report::models::{CostRun, CostSummary}; + +/// `_styled_cost_status`. +pub(crate) fn styled_status(style: CliStyle, status: &str) -> String { + if status == COMPLETE_STATUS { + return style.value(status); + } + if WARNING_STATUSES.contains(&status) { + return style.warning(status); + } + if status == COLLECTION_FAILED_STATUS { + return style.error(status); + } + style.muted(status) +} + +/// The credit rate, flagged when it is the default. +pub(crate) fn rate_note(style: CliStyle, cost: &CostSummary) -> String { + let rate: String = fixed(&cost.usd_per_credit, RATE_PLACES); + if cost.rate_source == DEFAULT_RATE_SOURCE { + return style.warning(&format!("default ${rate}/credit")); + } + style.muted(&format!("${rate}/credit")) +} + +/// The `Run ` line, when shown. +pub(crate) fn run_line(style: CliStyle, run: &CostRun, show: bool) -> Option { + show.then(|| format!("{} {}", style.label("Run"), run.run_id)) +} + +/// The overflow, message, rate, limitation and method notes after the table. +pub(crate) fn trailer(style: CliStyle, run: &CostRun, limit: Option) -> Vec { + let cost: &CostSummary = &run.cost; + let mut lines: Vec = Vec::new(); + if let Some(limit) = limit + && cost.resources.len() > limit + { + let total: usize = cost.resources.len(); + lines.push(style.muted(&format!( + "Showing {limit} of {total} by estimated cost; `sqb cost {}` for all models ({} more).", + run.run_id, + total - limit + ))); + } + if let Some(message) = cost.message.as_ref().filter(|message| !message.is_empty()) { + lines.push(style.muted(message)); + } + if cost.rate_source == DEFAULT_RATE_SOURCE { + lines.push(style.warning(RATE_ADVICE)); + } + lines.extend( + cost.limitations + .iter() + .map(|limitation| style.warning(limitation)), + ); + lines.push(style.muted(ESTIMATE_NOTE)); + lines +} diff --git a/crates/sqlbuild-core/src/cost_report/_helpers/decimal_text.rs b/crates/sqlbuild-core/src/cost_report/_helpers/decimal_text.rs new file mode 100644 index 0000000000..27f76969c4 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/_helpers/decimal_text.rs @@ -0,0 +1,125 @@ +//! Fixed-point text of Python `Decimal` values and byte sizes, rounded half to even. + +use crate::cost_report::constants::{BYTE_SCALE, BYTE_UNITS}; + +/// The digits and power of ten of a finite decimal text, with its sign. +struct DecimalParts { + negative: bool, + digits: String, + exponent: i64, +} + +fn parts(text: &str) -> Option { + let trimmed: &str = text.trim(); + let (negative, unsigned) = match trimmed.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, trimmed.strip_prefix('+').unwrap_or(trimmed)), + }; + let (mantissa, exponent) = match unsigned.find(['e', 'E']) { + Some(position) => { + let Ok(exponent) = unsigned[position + 1..].parse::() else { + return None; + }; + (&unsigned[..position], exponent) + } + None => (unsigned, 0), + }; + let (whole, fraction) = mantissa.split_once('.').unwrap_or((mantissa, "")); + let digits: String = format!("{whole}{fraction}"); + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let Ok(places) = i64::try_from(fraction.len()) else { + return None; + }; + Some(DecimalParts { + negative, + exponent: exponent - places, + digits, + }) +} + +/// Whether the text is a finite decimal Python's `Decimal` reads. +pub(crate) fn is_decimal(text: &str) -> bool { + parts(text).is_some() +} + +/// Round the digit string half to even after removing its last `dropped` digits. +fn rounded(digits: &str, dropped: usize) -> String { + if dropped == 0 { + return digits.to_owned(); + } + let padded: String = if dropped > digits.len() { + format!("{}{digits}", "0".repeat(dropped - digits.len())) + } else { + digits.to_owned() + }; + let (kept, tail) = padded.split_at(padded.len() - dropped); + let first: u8 = tail.as_bytes()[0]; + let rest_zero: bool = tail[1..].bytes().all(|byte| byte == b'0'); + let last_odd: bool = kept + .bytes() + .last() + .is_some_and(|byte| (byte - b'0') % 2 == 1); + let up: bool = first > b'5' || (first == b'5' && (!rest_zero || last_odd)); + let mut result: Vec = format!("0{kept}").into_bytes(); + if up { + for byte in result.iter_mut().rev() { + if *byte == b'9' { + *byte = b'0'; + } else { + *byte += 1; + break; + } + } + } + String::from_utf8(result).unwrap_or_default() +} + +/// `format(Decimal(text), f".{places}f")`; non-finite values keep Python's spelling. +pub(crate) fn fixed(text: &str, places: usize) -> String { + let Some(parts) = parts(text) else { + return text.trim().to_owned(); + }; + let scale: i64 = parts.exponent + i64::try_from(places).unwrap_or(i64::MAX); + let scaled: String = match usize::try_from(scale) { + Ok(zeros) => format!("{}{}", parts.digits, "0".repeat(zeros)), + Err(_) => rounded(&parts.digits, usize::try_from(-scale).unwrap_or(usize::MAX)), + }; + let padded: String = format!("{scaled:0>width$}", width = places + 1); + let (whole, fraction) = padded.split_at(padded.len() - places); + let whole: &str = whole.trim_start_matches('0'); + let whole: &str = if whole.is_empty() { "0" } else { whole }; + let sign: &str = if parts.negative { "-" } else { "" }; + if places == 0 { + return format!("{sign}{whole}"); + } + format!("{sign}{whole}.{fraction}") +} + +/// `_format_bytes`: whole bytes below a KiB, else one decimal of the largest fitting unit. +pub(crate) fn byte_size(value: i64) -> String { + let Ok(bytes) = u128::try_from(value) else { + return format!("{value} {}", BYTE_UNITS[0]); + }; + if bytes < BYTE_SCALE { + return format!("{value} {}", BYTE_UNITS[0]); + } + let mut divisor: u128 = BYTE_SCALE; + for (index, unit) in BYTE_UNITS.iter().enumerate().skip(1) { + if bytes < divisor * BYTE_SCALE || index == BYTE_UNITS.len() - 1 { + let numerator: u128 = bytes * 10; + let quotient: u128 = numerator / divisor; + let remainder: u128 = numerator % divisor; + let twice: u128 = remainder * 2; + let tenths: u128 = if twice > divisor || (twice == divisor && quotient % 2 == 1) { + quotient + 1 + } else { + quotient + }; + return format!("{}.{} {unit}", tenths / 10, tenths % 10); + } + divisor *= BYTE_SCALE; + } + format!("{value} {}", BYTE_UNITS[0]) +} diff --git a/crates/sqlbuild-core/src/cost_report/_helpers/history.rs b/crates/sqlbuild-core/src/cost_report/_helpers/history.rs new file mode 100644 index 0000000000..a99b9acca7 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/_helpers/history.rs @@ -0,0 +1,49 @@ +//! The unstyled rows of the cost history table. + +use crate::cost_report::_helpers::decimal_text::fixed; +use crate::cost_report::_helpers::instants::utc_text; +use crate::cost_report::_helpers::tables::{column_widths, padded, usd}; +use crate::cost_report::constants::{ + COLUMN_GAP, CREDIT_PLACES, HISTORY_HEADINGS, HISTORY_NUMERIC_COLUMNS, +}; +use crate::cost_report::models::CostRun; + +fn history_row(run: &CostRun) -> Vec { + vec![ + utc_text(&run.completed_at).unwrap_or_else(|| run.completed_at.clone()), + run.run_id.clone(), + run.build_status.clone(), + run.cost.status.clone(), + fixed(&run.cost.estimated_compute_credits, CREDIT_PLACES), + usd(&run.cost.estimated_usd), + ] +} + +fn aligned(row: &[String], widths: &[usize]) -> String { + let cells: Vec = row + .iter() + .enumerate() + .map(|(index, value)| { + padded( + value, + widths[index], + HISTORY_NUMERIC_COLUMNS.contains(&index), + ) + }) + .collect(); + cells.join(COLUMN_GAP) +} + +/// The heading line and one line per run. +pub(crate) fn history_rows(runs: &[CostRun]) -> Vec { + let rows: Vec> = runs.iter().map(history_row).collect(); + let widths: Vec = column_widths(&HISTORY_HEADINGS, &rows); + let headings: Vec = HISTORY_HEADINGS + .iter() + .enumerate() + .map(|(index, heading)| padded(heading, widths[index], false)) + .collect(); + let mut lines: Vec = vec![headings.join(COLUMN_GAP)]; + lines.extend(rows.iter().map(|row| aligned(row, &widths))); + lines +} diff --git a/crates/sqlbuild-core/src/cost_report/_helpers/instants.rs b/crates/sqlbuild-core/src/cost_report/_helpers/instants.rs new file mode 100644 index 0000000000..6e90354267 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/_helpers/instants.rs @@ -0,0 +1,172 @@ +//! The UTC instant of an ISO 8601 timestamp `datetime.isoformat()` wrote. + +use crate::cost_report::constants::{ + DAYS_PER_ERA, LAST_SHIFTED_MONTH, MAX_HOUR, MAX_MINUTE, MAX_MONTH, MICROS_PER_SECOND, + MICROSECOND_DIGITS, SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE, SHIFTED_JANUARY, + UNIX_EPOCH_DAYS_FROM_ERA_START, YEARS_PER_ERA, +}; + +/// A parsed timestamp: its wall-clock fields and its UTC offset, when it has one. +pub(crate) struct Timestamp { + /// Microseconds since the Unix epoch of the wall-clock time read as UTC. + pub(crate) local_micros: i128, + pub(crate) offset_seconds: Option, +} + +fn number(text: &[u8], count: usize) -> Option<(u32, &[u8])> { + if text.len() < count || !text[..count].iter().all(u8::is_ascii_digit) { + return None; + } + let value: u32 = text[..count] + .iter() + .fold(0, |total, digit| total * 10 + u32::from(digit - b'0')); + Some((value, &text[count..])) +} + +fn leap(year: i64) -> bool { + (year % 4 == 0 && year % 100 != 0) || year % YEARS_PER_ERA == 0 +} + +fn month_days(year: i64, month: u32) -> u32 { + match month { + 2 if leap(year) => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} + +/// Days since 1970-01-01 of a proleptic Gregorian date. +fn epoch_days(year: i64, month: u32, day: u32) -> i64 { + let shifted_year: i64 = if month <= LAST_SHIFTED_MONTH { + year - 1 + } else { + year + }; + let era: i64 = shifted_year.div_euclid(YEARS_PER_ERA); + let year_of_era: i64 = shifted_year - era * YEARS_PER_ERA; + let shifted_month: i64 = (i64::from(month) + 9) % 12; + let day_of_year: i64 = (153 * shifted_month + 2) / 5 + i64::from(day) - 1; + let day_of_era: i64 = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * DAYS_PER_ERA + day_of_era - UNIX_EPOCH_DAYS_FROM_ERA_START +} + +/// The civil date of days since 1970-01-01. +fn civil(days: i64) -> (i64, u32, u32) { + let shifted: i64 = days + UNIX_EPOCH_DAYS_FROM_ERA_START; + let era: i64 = shifted.div_euclid(DAYS_PER_ERA); + let day_of_era: i64 = shifted - era * DAYS_PER_ERA; + let year_of_era: i64 = + (day_of_era - day_of_era / 1460 + day_of_era / 36524 - day_of_era / 146_096) / 365; + let day_of_year: i64 = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let shifted_month: i64 = (5 * day_of_year + 2) / 153; + let day: u32 = u32::try_from(day_of_year - (153 * shifted_month + 2) / 5 + 1).unwrap_or(1); + let month: u32 = u32::try_from(if shifted_month < SHIFTED_JANUARY { + shifted_month + 3 + } else { + shifted_month - 9 + }) + .unwrap_or(1); + let year: i64 = year_of_era + era * YEARS_PER_ERA + i64::from(month <= LAST_SHIFTED_MONTH); + (year, month, day) +} + +/// `HH[:MM[:SS[.fff|.ffffff]]]` as seconds and microseconds, with the rest of the text. +fn clock(text: &[u8]) -> Option<(i128, i128, &[u8])> { + let (hour, mut rest) = number(text, 2)?; + let mut seconds: i128 = i128::from(hour) * SECONDS_PER_HOUR; + if hour > MAX_HOUR { + return None; + } + for scale in [SECONDS_PER_MINUTE, 1] { + let Some(after) = rest.strip_prefix(b":") else { + break; + }; + let (value, after) = number(after, 2)?; + if value > MAX_MINUTE { + return None; + } + seconds += i128::from(value) * scale; + rest = after; + } + let mut micros: i128 = 0; + if let Some(after) = rest.strip_prefix(b".") { + let length: usize = after + .iter() + .take_while(|byte| byte.is_ascii_digit()) + .count(); + if length == 0 || length > MICROSECOND_DIGITS { + return None; + } + let (value, after) = number(after, length)?; + let Ok(missing) = u32::try_from(MICROSECOND_DIGITS - length) else { + return None; + }; + micros = i128::from(value) * 10_i128.pow(missing); + rest = after; + } + Some((seconds, micros, rest)) +} + +fn offset(text: &[u8]) -> Option> { + if text.is_empty() { + return Some(None); + } + if text == b"Z" { + return Some(Some(0)); + } + let (sign, rest): (i128, &[u8]) = match text.first()? { + b'+' => (1, &text[1..]), + b'-' => (-1, &text[1..]), + _ => return None, + }; + let (seconds, micros, rest) = clock(rest)?; + if !rest.is_empty() || micros != 0 { + return None; + } + Some(Some(sign * seconds)) +} + +/// `datetime.fromisoformat` for the forms `isoformat()` writes. +pub(crate) fn parse(text: &str) -> Option { + let bytes: &[u8] = text.as_bytes(); + let (year, rest) = number(bytes, 4)?; + let (month, rest) = number(rest.strip_prefix(b"-")?, 2)?; + let (day, rest) = number(rest.strip_prefix(b"-")?, 2)?; + let year: i64 = i64::from(year); + if !(1..=MAX_MONTH).contains(&month) || day == 0 || day > month_days(year, month) { + return None; + } + let days: i128 = i128::from(epoch_days(year, month, day)); + let (seconds, micros, rest) = match rest.first() { + None => (0, 0, rest), + Some(b'T' | b' ') => clock(&rest[1..])?, + Some(_) => return None, + }; + Some(Timestamp { + local_micros: (days * SECONDS_PER_DAY + seconds) * MICROS_PER_SECOND + micros, + offset_seconds: offset(rest)?, + }) +} + +/// Microseconds since the Unix epoch, for an aware timestamp. +pub(crate) fn utc_micros(text: &str) -> Option { + let timestamp: Timestamp = parse(text)?; + Some(timestamp.local_micros - timestamp.offset_seconds? * MICROS_PER_SECOND) +} + +/// `astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S")` of an aware timestamp. +pub(crate) fn utc_text(text: &str) -> Option { + let seconds: i128 = utc_micros(text)?.div_euclid(MICROS_PER_SECOND); + let Ok(days) = i64::try_from(seconds.div_euclid(SECONDS_PER_DAY)) else { + return None; + }; + let clock: i128 = seconds.rem_euclid(SECONDS_PER_DAY); + let (year, month, day) = civil(days); + Some(format!( + "{year:04}-{month:02}-{day:02} {:02}:{:02}:{:02}", + clock / SECONDS_PER_HOUR, + clock % SECONDS_PER_HOUR / SECONDS_PER_MINUTE, + clock % SECONDS_PER_MINUTE + )) +} diff --git a/crates/sqlbuild-core/src/cost_report/_helpers/mod.rs b/crates/sqlbuild-core/src/cost_report/_helpers/mod.rs new file mode 100644 index 0000000000..95732ce753 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod breakdown; +pub(crate) mod decimal_text; +pub(crate) mod history; +pub(crate) mod instants; +pub(crate) mod parsing; +pub(crate) mod payloads; +pub(crate) mod tables; diff --git a/crates/sqlbuild-core/src/cost_report/_helpers/parsing.rs b/crates/sqlbuild-core/src/cost_report/_helpers/parsing.rs new file mode 100644 index 0000000000..bca3dff5f5 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/_helpers/parsing.rs @@ -0,0 +1,231 @@ +//! A persisted cost run artifact read with the coercions `CostRunRecord` loading applied. + +use crate::cost_report::_helpers::decimal_text::is_decimal; +use crate::cost_report::_helpers::instants::parse; +use crate::cost_report::constants::{ + ARTIFACT_VERSION, COST_STATUSES, RUN_ID_SEPARATORS, UNSAFE_RUN_IDS, +}; +use crate::cost_report::errors::CostArtifactError; +use crate::cost_report::models::{CostResource, CostRun, CostSummary}; +use crate::json::main::python_float_repr::python_float_repr; +use crate::json::models::{JsonInteger, JsonValue}; + +type Entries = [(String, JsonValue)]; + +fn invalid(message: impl Into) -> CostArtifactError { + CostArtifactError::new(message) +} + +fn get<'a>(entries: &'a Entries, key: &str) -> Option<&'a JsonValue> { + entries + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value) +} + +/// `payload[key]`: the value, or the failure of a missing key. +fn required<'a>(entries: &'a Entries, key: &str) -> Result<&'a JsonValue, CostArtifactError> { + get(entries, key).ok_or_else(|| invalid(format!("cost run artifact is missing '{key}'"))) +} + +/// `str(value)` of a JSON scalar. +fn text(value: &JsonValue) -> Result { + match value { + JsonValue::String(text) => Ok(text.clone()), + JsonValue::Integer(number) => Ok(number.as_str().to_owned()), + JsonValue::Float(number) => Ok(python_float_repr(*number)), + JsonValue::Bool(flag) => Ok(if *flag { "True" } else { "False" }.to_owned()), + JsonValue::Null => Ok("None".to_owned()), + _ => Err(invalid("cost run artifact value must be a scalar")), + } +} + +/// `None` for an absent or `null` value, else `str(value)`. +fn optional_text(value: Option<&JsonValue>) -> Result, CostArtifactError> { + match value { + None | Some(JsonValue::Null) => Ok(None), + Some(value) => text(value).map(Some), + } +} + +fn whole_number(text: &str) -> Option { + let Ok(number) = text.trim().parse::() else { + return None; + }; + Some(number) +} + +fn json_integer(number: &JsonInteger) -> Option { + let Ok(number) = i64::try_from(number.to_i128()?) else { + return None; + }; + Some(number) +} + +/// `int(value)` of an integer, boolean or integral string. +fn integer(value: &JsonValue) -> Result { + let parsed: Option = match value { + JsonValue::Integer(number) => json_integer(number), + JsonValue::Bool(flag) => Some(i64::from(*flag)), + JsonValue::String(text) => whole_number(text), + _ => None, + }; + parsed.ok_or_else(|| invalid("cost run artifact count must be an integer")) +} + +fn integer_or_zero(value: Option<&JsonValue>) -> Result { + value.map_or(Ok(0), integer) +} + +/// `str(value)` when `Decimal(str(value))` accepts it. +fn decimal(value: &JsonValue) -> Result { + let text: String = text(value)?; + if !is_decimal(&text) { + return Err(invalid("cost run artifact amount must be a decimal")); + } + Ok(text.trim().to_owned()) +} + +fn decimal_or_zero(value: Option<&JsonValue>) -> Result { + value.map_or(Ok("0".to_owned()), decimal) +} + +fn object<'a>(value: &'a JsonValue, message: &str) -> Result<&'a Entries, CostArtifactError> { + match value { + JsonValue::Object(entries) => Ok(entries), + _ => Err(invalid(message)), + } +} + +/// `_validate_run_id`: a run ID is exactly one safe path component. +pub(crate) fn check_run_id(run_id: &str) -> Result<(), CostArtifactError> { + if run_id.is_empty() || UNSAFE_RUN_IDS.contains(&run_id) || run_id.contains(RUN_ID_SEPARATORS) { + return Err(invalid("cost run ID must be one path component")); + } + Ok(()) +} + +fn resource(entries: &Entries) -> Result { + Ok(CostResource { + resource_type: text(required(entries, "resource_type")?)?, + resource_name: text(required(entries, "resource_name")?)?, + warehouse_name: text(required(entries, "warehouse_name")?)?, + warehouse_size: text(required(entries, "warehouse_size")?)?, + warehouse_type: optional_text(get(entries, "warehouse_type"))?, + cluster_number: match get(entries, "cluster_number") { + None | Some(JsonValue::Null) => None, + Some(value) => Some(integer(value)?), + }, + query_count: integer(required(entries, "query_count")?)?, + attributed_seconds: decimal(required(entries, "attributed_seconds")?)?, + bytes_scanned: integer(required(entries, "bytes_scanned")?)?, + estimated_compute_credits: decimal(required(entries, "estimated_compute_credits")?)?, + estimated_usd: decimal(required(entries, "estimated_usd")?)?, + }) +} + +fn resources(cost: &Entries) -> Result, CostArtifactError> { + let items: &[JsonValue] = match get(cost, "resources") { + None => &[], + Some(JsonValue::Array(items)) => items, + Some(_) => return Err(invalid("cost run artifact 'resources' must be a list")), + }; + let mut objects: Vec<&Entries> = Vec::with_capacity(items.len()); + for item in items { + objects.push(object(item, "cost run artifact resources must be objects")?); + } + objects.into_iter().map(resource).collect() +} + +fn limitations(cost: &Entries) -> Result, CostArtifactError> { + match get(cost, "limitations") { + None => Ok(Vec::new()), + Some(JsonValue::Array(items)) => items.iter().map(text).collect(), + Some(_) => Err(invalid("cost run artifact 'limitations' must be a list")), + } +} + +fn summary(cost: &Entries, resources: Vec) -> Result { + let status: String = text(required(cost, "status")?)?; + if !COST_STATUSES.contains(&status.as_str()) { + return Err(invalid(format!("unsupported cost status: {status}"))); + } + Ok(CostSummary { + status, + usd_per_credit: decimal(required(cost, "usd_per_credit")?)?, + rate_source: text(required(cost, "rate_source")?)?, + resources, + query_count: integer_or_zero(get(cost, "query_count"))?, + attributed_seconds: decimal_or_zero(get(cost, "attributed_seconds"))?, + bytes_scanned: integer_or_zero(get(cost, "bytes_scanned"))?, + estimated_compute_credits: decimal_or_zero(get(cost, "estimated_compute_credits"))?, + estimated_usd: decimal_or_zero(get(cost, "estimated_usd"))?, + expected_statement_count: integer_or_zero(get(cost, "expected_statement_count"))?, + observed_statement_count: integer_or_zero(get(cost, "observed_statement_count"))?, + missing_statement_count: integer_or_zero(get(cost, "missing_statement_count"))?, + skipped_statement_count: integer_or_zero(get(cost, "skipped_statement_count"))?, + source: optional_text(get(cost, "source"))?.unwrap_or_default(), + method: optional_text(get(cost, "method"))?.unwrap_or_default(), + limitations: limitations(cost)?, + message: optional_text(get(cost, "message"))?, + }) +} + +/// `_aware_datetime`: the timestamp text, which must carry a UTC offset. +fn aware(value: &JsonValue, field: &str) -> Result { + let text: String = text(value)?; + let Some(timestamp) = parse(&text) else { + return Err(invalid(format!( + "cost run artifact '{field}' must be an ISO timestamp" + ))); + }; + if timestamp.offset_seconds.is_none() { + return Err(invalid(format!( + "cost run artifact '{field}' must include a timezone" + ))); + } + Ok(text) +} + +fn executable_work(payload: &Entries) -> Result, CostArtifactError> { + match get(payload, "had_executable_work") { + None | Some(JsonValue::Null) => Ok(None), + Some(JsonValue::Bool(flag)) => Ok(Some(*flag)), + Some(_) => Err(invalid( + "cost run artifact 'had_executable_work' must be a boolean", + )), + } +} + +/// `_record_from_payload`: the run an artifact payload describes. +pub(crate) fn cost_run(payload: &JsonValue) -> Result { + let payload: &Entries = object(payload, "cost run artifact must be an object")?; + let version: i64 = integer(required(payload, "version")?)?; + if version != ARTIFACT_VERSION { + return Err(invalid(format!( + "unsupported cost run artifact version: {version}" + ))); + } + let run_id: String = text(required(payload, "run_id")?)?; + check_run_id(&run_id)?; + let cost: &Entries = object( + required(payload, "cost")?, + "cost run artifact 'cost' must be an object", + )?; + let resources: Vec = resources(cost)?; + let started_at: String = aware(required(payload, "started_at")?, "started_at")?; + let completed_at: String = aware(required(payload, "completed_at")?, "completed_at")?; + let had_executable_work: Option = executable_work(payload)?; + let cost: CostSummary = summary(cost, resources)?; + Ok(CostRun { + version, + run_id, + adapter_name: text(required(payload, "adapter_name")?)?, + target_name: optional_text(get(payload, "target_name"))?, + build_status: text(required(payload, "build_status")?)?, + started_at, + completed_at, + cost, + had_executable_work, + }) +} diff --git a/crates/sqlbuild-core/src/cost_report/_helpers/payloads.rs b/crates/sqlbuild-core/src/cost_report/_helpers/payloads.rs new file mode 100644 index 0000000000..74d6c0ad31 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/_helpers/payloads.rs @@ -0,0 +1,224 @@ +//! JSON documents of a cost run: the persisted artifact and the command output. + +use crate::cost_report::constants::OUTPUT_SCHEMA_VERSION; +use crate::cost_report::models::{CostResource, CostRun, CostSummary}; +use crate::json::models::{JsonInteger, JsonValue}; + +type Entries = Vec<(String, JsonValue)>; + +fn string(text: &str) -> JsonValue { + JsonValue::String(text.to_owned()) +} + +fn optional(text: Option<&String>) -> JsonValue { + text.map_or(JsonValue::Null, |text| string(text)) +} + +fn integer(value: i64) -> JsonValue { + JsonValue::Integer(JsonInteger::from(value)) +} + +fn object(entries: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + entries + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + ) +} + +fn strings(items: &[String]) -> JsonValue { + JsonValue::Array(items.iter().map(|item| string(item)).collect()) +} + +fn optional_flag(flag: Option) -> JsonValue { + flag.map_or(JsonValue::Null, JsonValue::Bool) +} + +fn optional_integer(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, integer) +} + +fn stored_resource(resource: &CostResource) -> JsonValue { + object(vec![ + ("resource_type", string(&resource.resource_type)), + ("resource_name", string(&resource.resource_name)), + ("warehouse_name", string(&resource.warehouse_name)), + ("warehouse_size", string(&resource.warehouse_size)), + ("warehouse_type", optional(resource.warehouse_type.as_ref())), + ("cluster_number", optional_integer(resource.cluster_number)), + ("query_count", integer(resource.query_count)), + ("attributed_seconds", string(&resource.attributed_seconds)), + ("bytes_scanned", integer(resource.bytes_scanned)), + ( + "estimated_compute_credits", + string(&resource.estimated_compute_credits), + ), + ("estimated_usd", string(&resource.estimated_usd)), + ]) +} + +fn stored_summary(cost: &CostSummary) -> JsonValue { + object(vec![ + ("status", string(&cost.status)), + ("usd_per_credit", string(&cost.usd_per_credit)), + ("rate_source", string(&cost.rate_source)), + ( + "resources", + JsonValue::Array(cost.resources.iter().map(stored_resource).collect()), + ), + ("query_count", integer(cost.query_count)), + ("attributed_seconds", string(&cost.attributed_seconds)), + ("bytes_scanned", integer(cost.bytes_scanned)), + ( + "estimated_compute_credits", + string(&cost.estimated_compute_credits), + ), + ("estimated_usd", string(&cost.estimated_usd)), + ( + "expected_statement_count", + integer(cost.expected_statement_count), + ), + ( + "observed_statement_count", + integer(cost.observed_statement_count), + ), + ( + "missing_statement_count", + integer(cost.missing_statement_count), + ), + ( + "skipped_statement_count", + integer(cost.skipped_statement_count), + ), + ("source", string(&cost.source)), + ("method", string(&cost.method)), + ("limitations", strings(&cost.limitations)), + ("message", optional(cost.message.as_ref())), + ]) +} + +/// `asdict(record)` with decimals and timestamps as text, as the artifact stores it. +pub(crate) fn artifact(run: &CostRun) -> JsonValue { + object(vec![ + ("run_id", string(&run.run_id)), + ("adapter_name", string(&run.adapter_name)), + ("target_name", optional(run.target_name.as_ref())), + ("build_status", string(&run.build_status)), + ("started_at", string(&run.started_at)), + ("completed_at", string(&run.completed_at)), + ("cost", stored_summary(&run.cost)), + ("version", integer(run.version)), + ( + "had_executable_work", + optional_flag(run.had_executable_work), + ), + ]) +} + +fn output_resource(resource: &CostResource) -> JsonValue { + object(vec![ + ("resource_type", string(&resource.resource_type)), + ("model", string(&resource.resource_name)), + ("warehouse", string(&resource.warehouse_name)), + ("warehouse_size", string(&resource.warehouse_size)), + ("warehouse_type", optional(resource.warehouse_type.as_ref())), + ("cluster_number", optional_integer(resource.cluster_number)), + ("query_count", integer(resource.query_count)), + ("attributed_seconds", string(&resource.attributed_seconds)), + ("bytes_scanned", integer(resource.bytes_scanned)), + ( + "attributed_compute_credits", + string(&resource.estimated_compute_credits), + ), + ("estimated_usd", string(&resource.estimated_usd)), + ]) +} + +fn coverage(cost: &CostSummary) -> JsonValue { + object(vec![ + ("query_count", integer(cost.query_count)), + ( + "model_rows", + integer(i64::try_from(cost.resources.len()).unwrap_or(i64::MAX)), + ), + ( + "expected_statement_count", + integer(cost.expected_statement_count), + ), + ( + "observed_statement_count", + integer(cost.observed_statement_count), + ), + ( + "missing_statement_count", + integer(cost.missing_statement_count), + ), + ( + "skipped_statement_count", + integer(cost.skipped_statement_count), + ), + ]) +} + +fn totals(cost: &CostSummary) -> JsonValue { + object(vec![ + ("query_count", integer(cost.query_count)), + ("attributed_seconds", string(&cost.attributed_seconds)), + ("bytes_scanned", integer(cost.bytes_scanned)), + ( + "attributed_compute_credits", + string(&cost.estimated_compute_credits), + ), + ("estimated_usd", string(&cost.estimated_usd)), + ]) +} + +/// `cost_output_payload`: the `sqb cost --json` document of one run. +pub(crate) fn output(run: &CostRun) -> JsonValue { + let cost: &CostSummary = &run.cost; + let entries: Entries = vec![ + ("schema_version".to_owned(), integer(OUTPUT_SCHEMA_VERSION)), + ("command".to_owned(), string("cost")), + ("run_id".to_owned(), string(&run.run_id)), + ("build_status".to_owned(), string(&run.build_status)), + ("started_at".to_owned(), string(&run.started_at)), + ("completed_at".to_owned(), string(&run.completed_at)), + ("adapter".to_owned(), string(&run.adapter_name)), + ("target".to_owned(), optional(run.target_name.as_ref())), + ( + "had_executable_work".to_owned(), + optional_flag(run.had_executable_work), + ), + ("status".to_owned(), string(&cost.status)), + ("source".to_owned(), string(&cost.source)), + ("currency".to_owned(), string("USD")), + ("usd_per_credit".to_owned(), string(&cost.usd_per_credit)), + ("rate_source".to_owned(), string(&cost.rate_source)), + ("method".to_owned(), string(&cost.method)), + ("limitations".to_owned(), strings(&cost.limitations)), + ("message".to_owned(), optional(cost.message.as_ref())), + ("coverage".to_owned(), coverage(cost)), + ( + "per_model".to_owned(), + JsonValue::Array(cost.resources.iter().map(output_resource).collect()), + ), + ("totals".to_owned(), totals(cost)), + ]; + JsonValue::Object(entries) +} + +/// `format_cost_history_json`'s document of several runs. +pub(crate) fn history(runs: &[CostRun], matching_count: Option) -> JsonValue { + let count: i64 = i64::try_from(runs.len()).unwrap_or(i64::MAX); + let matching: i64 = matching_count.map_or(count, |matching| { + i64::try_from(matching).unwrap_or(i64::MAX) + }); + object(vec![ + ("schema_version", integer(OUTPUT_SCHEMA_VERSION)), + ("command", string("cost_history")), + ("count", integer(count)), + ("matching_count", integer(matching)), + ("runs", JsonValue::Array(runs.iter().map(output).collect())), + ]) +} diff --git a/crates/sqlbuild-core/src/cost_report/_helpers/tables.rs b/crates/sqlbuild-core/src/cost_report/_helpers/tables.rs new file mode 100644 index 0000000000..7c1fd3b674 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/_helpers/tables.rs @@ -0,0 +1,151 @@ +//! Aligned cost tables: column widths, padding and truncated names. + +use crate::cli_text::models::CliStyle; +use crate::cost_report::_helpers::decimal_text::{byte_size, fixed}; +use crate::cost_report::constants::{ + COLUMN_GAP, CREDIT_PLACES, ELLIPSIS, NARROW_HEADINGS, NARROW_NAME_LIMIT, NARROW_TERMINAL_WIDTH, + RESOURCE_NUMERIC_START_COLUMN, SECONDS_PLACES, TOTAL_LABEL, USD_PLACES, WAREHOUSE_NAME_LIMIT, + WIDE_HEADINGS, WIDE_NAME_LIMIT, +}; +use crate::cost_report::models::{CostResource, CostSummary}; + +fn width(text: &str) -> usize { + text.chars().count() +} + +/// `value.ljust(width)` or `value.rjust(width)`. +pub(crate) fn padded(value: &str, columns: usize, right: bool) -> String { + let fill: String = " ".repeat(columns.saturating_sub(width(value))); + if right { + format!("{fill}{value}") + } else { + format!("{value}{fill}") + } +} + +/// `_truncate`: the text, or its prefix and an ellipsis within `limit` characters. +fn truncated(value: &str, limit: usize) -> String { + if width(value) <= limit { + return value.to_owned(); + } + let prefix: String = value + .chars() + .take(limit.saturating_sub(ELLIPSIS.len())) + .collect(); + format!("{prefix}{ELLIPSIS}") +} + +/// `_format_usd`. +pub(crate) fn usd(value: &str) -> String { + format!("${}", fixed(value, USD_PLACES)) +} + +fn column_width(heading: &str, rows: &[Vec], index: usize) -> usize { + let mut widest: usize = width(heading); + for row in rows { + widest = widest.max(width(&row[index])); + } + widest +} + +/// `_column_widths`: the widest heading or cell of every column. +pub(crate) fn column_widths(headings: &[&str], rows: &[Vec]) -> Vec { + headings + .iter() + .enumerate() + .map(|(index, heading)| column_width(heading, rows, index)) + .collect() +} + +fn styled_row( + style: CliStyle, + row: &[String], + (widths, numeric_start): (&[usize], usize), +) -> String { + let cells: Vec = row + .iter() + .enumerate() + .map(|(index, value)| { + let cell: String = padded(value, widths[index], index >= numeric_start); + if index == 0 { + style.object_name(&cell) + } else { + style.value(&cell) + } + }) + .collect(); + cells.join(COLUMN_GAP) +} + +/// `_format_table_rows`: a styled heading line and one styled line per row. +fn table( + style: CliStyle, + headings: &[&str], + rows: &[Vec], + numeric_start: usize, +) -> Vec { + let widths: Vec = column_widths(headings, rows); + let heading: Vec = headings + .iter() + .enumerate() + .map(|(index, heading)| padded(heading, widths[index], false)) + .collect(); + let mut lines: Vec = vec![style.label(&heading.join(COLUMN_GAP))]; + lines.extend( + rows.iter() + .map(|row| styled_row(style, row, (&widths, numeric_start))), + ); + lines +} + +fn narrow_row(resource: &CostResource) -> Vec { + vec![ + truncated(&resource.resource_name, NARROW_NAME_LIMIT), + fixed(&resource.estimated_compute_credits, CREDIT_PLACES), + usd(&resource.estimated_usd), + ] +} + +fn wide_row(resource: &CostResource) -> Vec { + vec![ + truncated(&resource.resource_name, WIDE_NAME_LIMIT), + truncated(&resource.warehouse_name, WAREHOUSE_NAME_LIMIT), + format!("{}s", fixed(&resource.attributed_seconds, SECONDS_PLACES)), + byte_size(resource.bytes_scanned), + fixed(&resource.estimated_compute_credits, CREDIT_PLACES), + usd(&resource.estimated_usd), + ] +} + +/// `_format_resource_rows`: the narrow or wide resource table with its total. +pub(crate) fn resource_rows( + style: CliStyle, + resources: &[CostResource], + total: &CostSummary, + terminal_width: usize, +) -> Vec { + if terminal_width < NARROW_TERMINAL_WIDTH { + let mut rows: Vec> = resources.iter().map(narrow_row).collect(); + rows.push(vec![ + TOTAL_LABEL.to_owned(), + fixed(&total.estimated_compute_credits, CREDIT_PLACES), + usd(&total.estimated_usd), + ]); + return table( + style, + &NARROW_HEADINGS, + &rows, + RESOURCE_NUMERIC_START_COLUMN - 1, + ); + } + let mut rows: Vec> = resources.iter().map(wide_row).collect(); + rows.push(vec![ + TOTAL_LABEL.to_owned(), + String::new(), + format!("{}s", fixed(&total.attributed_seconds, SECONDS_PLACES)), + byte_size(total.bytes_scanned), + fixed(&total.estimated_compute_credits, CREDIT_PLACES), + usd(&total.estimated_usd), + ]); + table(style, &WIDE_HEADINGS, &rows, RESOURCE_NUMERIC_START_COLUMN) +} diff --git a/crates/sqlbuild-core/src/cost_report/constants.rs b/crates/sqlbuild-core/src/cost_report/constants.rs new file mode 100644 index 0000000000..2be6918ced --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/constants.rs @@ -0,0 +1,69 @@ +//! Cost artifact identity, report wording and table layout. + +pub(crate) const ARTIFACT_VERSION: i64 = 1; +pub(crate) const OUTPUT_SCHEMA_VERSION: i64 = 1; +pub(crate) const LATEST_SELECTOR: &str = "latest"; +pub(crate) const RUNNING_BUILD_STATUS: &str = "running"; +pub(crate) const DEFAULT_RATE_SOURCE: &str = "default"; +pub(crate) const COMPLETE_STATUS: &str = "complete"; +pub(crate) const COLLECTION_FAILED_STATUS: &str = "collection_failed"; +pub(crate) const WARNING_STATUSES: [&str; 2] = ["partial", "pending"]; +pub(crate) const COST_STATUSES: [&str; 5] = [ + "complete", + "partial", + "pending", + "unavailable", + "collection_failed", +]; +pub(crate) const UNSAFE_RUN_IDS: [&str; 2] = [".", ".."]; +pub(crate) const RUN_ID_SEPARATORS: [char; 2] = ['/', '\\']; +pub(crate) const NARROW_TERMINAL_WIDTH: usize = 120; +pub(crate) const HISTORY_NUMERIC_COLUMNS: [usize; 2] = [4, 5]; +pub(crate) const RESOURCE_NUMERIC_START_COLUMN: usize = 2; +pub(crate) const BYTE_SCALE: u128 = 1024; +pub(crate) const BYTE_UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; +pub(crate) const NARROW_NAME_LIMIT: usize = 32; +pub(crate) const WIDE_NAME_LIMIT: usize = 36; +pub(crate) const WAREHOUSE_NAME_LIMIT: usize = 20; +pub(crate) const ELLIPSIS: &str = "..."; +pub(crate) const USD_PLACES: usize = 4; +pub(crate) const CREDIT_PLACES: usize = 6; +pub(crate) const SECONDS_PLACES: usize = 2; +pub(crate) const RATE_PLACES: usize = 2; +pub(crate) const TOTAL_LABEL: &str = "TOTAL"; +pub(crate) const COLUMN_GAP: &str = " "; +pub(crate) const HISTORY_HEADINGS: [&str; 6] = [ + "Completed (UTC)", + "Run", + "Build", + "Cost", + "Attributed credits", + "Est. USD", +]; +pub(crate) const NARROW_HEADINGS: [&str; 3] = ["Model", "Attributed credits", "Est. cost"]; +pub(crate) const WIDE_HEADINGS: [&str; 6] = [ + "Model", + "Warehouse", + "Busy share", + "Scanned", + "Attributed credits", + "Est. cost", +]; +pub(crate) const RATE_ADVICE: &str = + "Configure cost.usd_per_credit with your Snowflake contract rate."; +pub(crate) const ESTIMATE_NOTE: &str = "Estimate uses fair-share visible warehouse busy time; it is not Snowflake-billed credits or invoice reconciliation."; +pub(crate) const MICROS_PER_SECOND: i128 = 1_000_000; +pub(crate) const SECONDS_PER_DAY: i128 = 86_400; +pub(crate) const SECONDS_PER_HOUR: i128 = 3_600; +pub(crate) const SECONDS_PER_MINUTE: i128 = 60; +pub(crate) const MAX_HOUR: u32 = 23; +pub(crate) const MAX_MINUTE: u32 = 59; +pub(crate) const MAX_MONTH: u32 = 12; +pub(crate) const MICROSECOND_DIGITS: usize = 6; +pub(crate) const DAYS_PER_ERA: i64 = 146_097; +pub(crate) const YEARS_PER_ERA: i64 = 400; +pub(crate) const UNIX_EPOCH_DAYS_FROM_ERA_START: i64 = 719_468; +/// January and February count as the end of the previous year in the civil-day algorithm. +pub(crate) const LAST_SHIFTED_MONTH: u32 = 2; +/// The shifted month index of January (March is 0). +pub(crate) const SHIFTED_JANUARY: i64 = 10; diff --git a/crates/sqlbuild-core/src/cost_report/errors.rs b/crates/sqlbuild-core/src/cost_report/errors.rs new file mode 100644 index 0000000000..8677fb0d25 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/errors.rs @@ -0,0 +1,23 @@ +//! A persisted cost run that cannot be read. + +use std::fmt; + +/// The reason a cost run artifact is unusable, as `CostArtifactError` reports it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CostArtifactError { + pub message: String, +} + +impl CostArtifactError { + pub(crate) fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl fmt::Display for CostArtifactError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.message) + } +} diff --git a/crates/sqlbuild-core/src/cost_report/main/check_cost_run_id.rs b/crates/sqlbuild-core/src/cost_report/main/check_cost_run_id.rs new file mode 100644 index 0000000000..e97ae3b371 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/check_cost_run_id.rs @@ -0,0 +1,9 @@ +//! The path-safety rule of cost run IDs. + +use crate::cost_report::_helpers::parsing::check_run_id; +use crate::cost_report::errors::CostArtifactError; + +/// A run ID must be exactly one path component other than `.` and `..`. +pub fn check_cost_run_id(run_id: &str) -> Result<(), CostArtifactError> { + check_run_id(run_id) +} diff --git a/crates/sqlbuild-core/src/cost_report/main/cost_artifact_text.rs b/crates/sqlbuild-core/src/cost_report/main/cost_artifact_text.rs new file mode 100644 index 0000000000..016afe3460 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/cost_artifact_text.rs @@ -0,0 +1,16 @@ +//! The persisted `run.json` text of a cost run. + +use crate::cost_report::_helpers::payloads::artifact; +use crate::cost_report::models::CostRun; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, StdlibJsonOptions}; + +/// `json.dumps(asdict(record), indent=2, sort_keys=True) + "\n"`. +#[must_use] +pub fn cost_artifact_text(run: &CostRun) -> String { + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(Some(2)) + }; + dumps(&artifact(run), &JsonDialect::Stdlib(options)).unwrap_or_default() + "\n" +} diff --git a/crates/sqlbuild-core/src/cost_report/main/cost_history_json.rs b/crates/sqlbuild-core/src/cost_report/main/cost_history_json.rs new file mode 100644 index 0000000000..357bdfa4b4 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/cost_history_json.rs @@ -0,0 +1,20 @@ +//! `sqb cost history --json` text. + +use crate::cost_report::_helpers::payloads::history; +use crate::cost_report::models::CostRun; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, StdlibJsonOptions}; + +/// The history document of the shown runs; `matching_count` defaults to their number. +#[must_use] +pub fn cost_history_json(runs: &[CostRun], matching_count: Option) -> String { + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(Some(2)) + }; + dumps( + &history(runs, matching_count), + &JsonDialect::Stdlib(options), + ) + .unwrap_or_default() +} diff --git a/crates/sqlbuild-core/src/cost_report/main/cost_output_payload.rs b/crates/sqlbuild-core/src/cost_report/main/cost_output_payload.rs new file mode 100644 index 0000000000..b92d282a02 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/cost_output_payload.rs @@ -0,0 +1,11 @@ +//! The `sqb cost --json` document of one run. + +use crate::cost_report::_helpers::payloads::output; +use crate::cost_report::models::CostRun; +use crate::json::models::JsonValue; + +/// The run's output document, keys in emission order. +#[must_use] +pub fn cost_output_payload(run: &CostRun) -> JsonValue { + output(run) +} diff --git a/crates/sqlbuild-core/src/cost_report/main/cost_report_json.rs b/crates/sqlbuild-core/src/cost_report/main/cost_report_json.rs new file mode 100644 index 0000000000..6f550e280b --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/cost_report_json.rs @@ -0,0 +1,16 @@ +//! `sqb cost --json` text of one run. + +use crate::cost_report::_helpers::payloads::output; +use crate::cost_report::models::CostRun; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, StdlibJsonOptions}; + +/// `json.dumps(cost_output_payload(record), indent=2, sort_keys=True)`. +#[must_use] +pub fn cost_report_json(run: &CostRun) -> String { + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(Some(2)) + }; + dumps(&output(run), &JsonDialect::Stdlib(options)).unwrap_or_default() +} diff --git a/crates/sqlbuild-core/src/cost_report/main/format_cost_breakdown.rs b/crates/sqlbuild-core/src/cost_report/main/format_cost_breakdown.rs new file mode 100644 index 0000000000..45c8173e9c --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/format_cost_breakdown.rs @@ -0,0 +1,66 @@ +//! The human breakdown of one run's cost, shared by build output and `sqb cost`. + +use crate::cli_text::models::CliStyle; +use crate::cost_report::_helpers::breakdown::{rate_note, run_line, styled_status, trailer}; +use crate::cost_report::_helpers::decimal_text::fixed; +use crate::cost_report::_helpers::tables::{resource_rows, usd}; +use crate::cost_report::constants::{COMPLETE_STATUS, CREDIT_PLACES}; +use crate::cost_report::models::{BreakdownOptions, CostRun}; + +/// `format_cost_breakdown`: totals, status, the resource table and the run's notes. +#[must_use] +pub fn format_cost_breakdown(run: &CostRun, options: BreakdownOptions) -> String { + let style: CliStyle = CliStyle { + use_color: options.use_color, + }; + let cost = &run.cost; + let heading: String = style.title("Cost"); + let status: String = styled_status(style, &cost.status); + if cost.resources.is_empty() && cost.status != COMPLETE_STATUS { + let mut lines: Vec = vec![format!("{heading} {status}")]; + lines.extend(run_line(style, run, options.show_run_id)); + lines.extend( + cost.message + .iter() + .filter(|message| !message.is_empty()) + .map(|message| style.muted(message)), + ); + lines.extend( + cost.limitations + .iter() + .map(|limitation| style.warning(limitation)), + ); + lines.push(style.muted(&format!( + "Run `sqb cost {}` to inspect this run later.", + run.run_id + ))); + return lines.join("\n"); + } + let mut lines: Vec = vec![format!( + "{heading} {} estimated | {} | {}", + style.value(&usd(&cost.estimated_usd)), + style.value(&format!( + "{} attributed credits", + fixed(&cost.estimated_compute_credits, CREDIT_PLACES) + )), + rate_note(style, cost), + )]; + if cost.status != COMPLETE_STATUS { + lines.push(status); + } + lines.extend(run_line(style, run, options.show_run_id)); + let shown: usize = options.limit.map_or(cost.resources.len(), |limit| { + limit.min(cost.resources.len()) + }); + if shown > 0 { + lines.push(String::new()); + lines.extend(resource_rows( + style, + &cost.resources[..shown], + cost, + options.terminal_width, + )); + } + lines.extend(trailer(style, run, options.limit)); + lines.join("\n") +} diff --git a/crates/sqlbuild-core/src/cost_report/main/format_cost_history.rs b/crates/sqlbuild-core/src/cost_report/main/format_cost_history.rs new file mode 100644 index 0000000000..27f4e03e5c --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/format_cost_history.rs @@ -0,0 +1,27 @@ +//! The human `sqb cost history` table. + +use crate::cli_text::models::CliStyle; +use crate::cost_report::_helpers::history::history_rows; +use crate::cost_report::models::CostRun; + +/// `format_cost_history`: one aligned row per run, newest first as given. +#[must_use] +pub fn format_cost_history(runs: &[CostRun], matching_count: usize, use_color: bool) -> String { + let style: CliStyle = CliStyle { use_color }; + let title: String = style.title("SQLBuild cost history"); + if runs.is_empty() { + return [ + title, + style.muted("No persisted cost runs."), + style.muted(&format!("Showing 0 of {matching_count} matching runs.")), + ] + .join("\n"); + } + let mut lines: Vec = vec![title]; + lines.extend(history_rows(runs)); + lines.push(style.muted(&format!( + "Showing {} of {matching_count} matching runs. Run `sqb cost ` for a full breakdown.", + runs.len() + ))); + lines.join("\n") +} diff --git a/crates/sqlbuild-core/src/cost_report/main/mod.rs b/crates/sqlbuild-core/src/cost_report/main/mod.rs new file mode 100644 index 0000000000..7e161a6249 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/mod.rs @@ -0,0 +1,10 @@ +pub mod check_cost_run_id; +pub mod cost_artifact_text; +pub mod cost_history_json; +pub mod cost_output_payload; +pub mod cost_report_json; +pub mod format_cost_breakdown; +pub mod format_cost_history; +pub mod order_cost_runs; +pub mod read_cost_artifact; +pub mod select_cost_run; diff --git a/crates/sqlbuild-core/src/cost_report/main/order_cost_runs.rs b/crates/sqlbuild-core/src/cost_report/main/order_cost_runs.rs new file mode 100644 index 0000000000..44343dac75 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/order_cost_runs.rs @@ -0,0 +1,18 @@ +//! Persisted runs newest first. + +use std::cmp::Reverse; + +use crate::cost_report::_helpers::instants::utc_micros; +use crate::cost_report::models::CostRun; + +/// The runs by completion instant, then run ID, both descending. +#[must_use] +pub fn order_cost_runs(mut runs: Vec) -> Vec { + runs.sort_by_cached_key(|run| { + Reverse(( + utc_micros(&run.completed_at).unwrap_or(i128::MIN), + run.run_id.clone(), + )) + }); + runs +} diff --git a/crates/sqlbuild-core/src/cost_report/main/read_cost_artifact.rs b/crates/sqlbuild-core/src/cost_report/main/read_cost_artifact.rs new file mode 100644 index 0000000000..044bc188e9 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/read_cost_artifact.rs @@ -0,0 +1,11 @@ +//! A decoded `run.json` payload as a cost run. + +use crate::cost_report::_helpers::parsing::cost_run; +use crate::cost_report::errors::CostArtifactError; +use crate::cost_report::models::CostRun; +use crate::json::models::JsonValue; + +/// The run the payload describes, or why the artifact is unusable. +pub fn read_cost_artifact(payload: &JsonValue) -> Result { + cost_run(payload) +} diff --git a/crates/sqlbuild-core/src/cost_report/main/select_cost_run.rs b/crates/sqlbuild-core/src/cost_report/main/select_cost_run.rs new file mode 100644 index 0000000000..77cb42c9ce --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/select_cost_run.rs @@ -0,0 +1,19 @@ +//! The run a `sqb cost` selector names. + +use crate::cost_report::constants::{LATEST_SELECTOR, RUNNING_BUILD_STATUS}; +use crate::cost_report::models::CostRun; + +/// `latest` picks the newest finished run; otherwise the only run whose ID starts with the selector. +#[must_use] +pub fn select_cost_run(runs: Vec, selector: &str) -> Option { + if selector == LATEST_SELECTOR { + return runs + .into_iter() + .find(|run| run.build_status != RUNNING_BUILD_STATUS); + } + let mut matches = runs + .into_iter() + .filter(|run| run.run_id.starts_with(selector)); + let first: CostRun = matches.next()?; + matches.next().is_none().then_some(first) +} diff --git a/crates/sqlbuild-core/src/cost_report/mod.rs b/crates/sqlbuild-core/src/cost_report/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/cost_report/models.rs b/crates/sqlbuild-core/src/cost_report/models.rs new file mode 100644 index 0000000000..65f5deef25 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/models.rs @@ -0,0 +1,64 @@ +//! A persisted cost run as plain data; decimals and timestamps keep Python's text form. + +/// One resource row of a run's cost (`ResourceCost`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CostResource { + pub resource_type: String, + pub resource_name: String, + pub warehouse_name: String, + pub warehouse_size: String, + pub warehouse_type: Option, + pub cluster_number: Option, + pub query_count: i64, + /// `str(Decimal)` text. + pub attributed_seconds: String, + pub bytes_scanned: i64, + pub estimated_compute_credits: String, + pub estimated_usd: String, +} + +/// A run's cost (`RunCostSummary`); decimal fields are `str(Decimal)` text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CostSummary { + pub status: String, + pub usd_per_credit: String, + pub rate_source: String, + pub resources: Vec, + pub query_count: i64, + pub attributed_seconds: String, + pub bytes_scanned: i64, + pub estimated_compute_credits: String, + pub estimated_usd: String, + pub expected_statement_count: i64, + pub observed_statement_count: i64, + pub missing_statement_count: i64, + pub skipped_statement_count: i64, + pub source: String, + pub method: String, + pub limitations: Vec, + pub message: Option, +} + +/// One persisted run (`CostRunRecord`); timestamps are `datetime.isoformat()` text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CostRun { + pub run_id: String, + pub adapter_name: String, + pub target_name: Option, + pub build_status: String, + pub started_at: String, + pub completed_at: String, + pub cost: CostSummary, + pub version: i64, + pub had_executable_work: Option, +} + +/// How a run's breakdown is laid out. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BreakdownOptions { + pub use_color: bool, + /// Show at most this many resources. + pub limit: Option, + pub terminal_width: usize, + pub show_run_id: bool, +} diff --git a/crates/sqlbuild-core/src/cost_report/tests.rs b/crates/sqlbuild-core/src/cost_report/tests.rs new file mode 100644 index 0000000000..581d3f7814 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_cost_report.rs"] +mod cost_report; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/cost_report/tests/helpers.rs b/crates/sqlbuild-core/src/cost_report/tests/helpers.rs new file mode 100644 index 0000000000..67c9632d08 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/tests/helpers.rs @@ -0,0 +1,49 @@ +use crate::cost_report::models::{CostResource, CostRun, CostSummary}; + +pub(super) fn resource(name: &str, usd: &str, bytes_scanned: i64) -> CostResource { + CostResource { + resource_type: "model".to_owned(), + resource_name: name.to_owned(), + warehouse_name: "TRANSFORM_WH".to_owned(), + warehouse_size: "X-SMALL".to_owned(), + warehouse_type: Some("STANDARD".to_owned()), + cluster_number: Some(1), + query_count: 2, + attributed_seconds: "12.345".to_owned(), + bytes_scanned, + estimated_compute_credits: "0.0034291666666666666666666667".to_owned(), + estimated_usd: usd.to_owned(), + } +} + +pub(super) fn run(completed_at: &str, resources: Vec) -> CostRun { + CostRun { + run_id: "run-1".to_owned(), + adapter_name: "snowflake".to_owned(), + target_name: Some("dev".to_owned()), + build_status: "success".to_owned(), + started_at: "2026-09-01T10:00:00+02:00".to_owned(), + completed_at: completed_at.to_owned(), + cost: CostSummary { + status: "complete".to_owned(), + usd_per_credit: "3.00".to_owned(), + rate_source: "default".to_owned(), + resources, + query_count: 4, + attributed_seconds: "24.69".to_owned(), + bytes_scanned: 1536, + estimated_compute_credits: "0.0068583333333333333333333334".to_owned(), + estimated_usd: "0.0205750000000000000000000002".to_owned(), + expected_statement_count: 4, + observed_statement_count: 4, + missing_statement_count: 0, + skipped_statement_count: 0, + source: "snowflake_information_schema_query_history".to_owned(), + method: "equal_share_visible_query_busy_time_v1".to_owned(), + limitations: Vec::new(), + message: None, + }, + version: 1, + had_executable_work: Some(true), + } +} diff --git a/crates/sqlbuild-core/src/cost_report/tests/test_cost_report.rs b/crates/sqlbuild-core/src/cost_report/tests/test_cost_report.rs new file mode 100644 index 0000000000..9c2486e26c --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/tests/test_cost_report.rs @@ -0,0 +1,72 @@ +use crate::cost_report::main::format_cost_breakdown::format_cost_breakdown; +use crate::cost_report::main::format_cost_history::format_cost_history; +use crate::cost_report::models::BreakdownOptions; +use crate::cost_report::tests::helpers::{resource, run}; +use crate::cost_report::tests::test_types::{BreakdownTestCase, HistoryTestCase}; + +#[test] +fn given_cost_runs_when_formatting_breakdown_then_table_matches_python_layout() { + let test_cases = [ + BreakdownTestCase { + description: "a wide table rounds half to even and scales bytes", + run: run( + "2026-09-01T10:05:00+02:00", + vec![ + resource("orders", "0.0102875", 5 * 1024 * 1024 * 1024 * 1024 + 1), + resource("customers", "1E-7", 1536), + ], + ), + limit: None, + terminal_width: 140, + expected_text: "Cost $0.0206 estimated | 0.006858 attributed credits | default $3.00/credit\nRun run-1\n\nModel Warehouse Busy share Scanned Attributed credits Est. cost\norders TRANSFORM_WH 12.34s 5.0 TiB 0.003429 $0.0103\ncustomers TRANSFORM_WH 12.34s 1.5 KiB 0.003429 $0.0000\nTOTAL 24.69s 1.5 KiB 0.006858 $0.0206\nConfigure cost.usd_per_credit with your Snowflake contract rate.\nEstimate uses fair-share visible warehouse busy time; it is not Snowflake-billed credits or invoice reconciliation.", + }, + BreakdownTestCase { + description: "a narrow limited table notes the hidden resources", + run: run( + "2026-09-01T10:05:00+02:00", + vec![ + resource("orders_with_a_very_long_model_name_x", "2.5", 0), + resource("customers", "0.5", 0), + ], + ), + limit: Some(1), + terminal_width: 80, + expected_text: "Cost $0.0206 estimated | 0.006858 attributed credits | default $3.00/credit\nRun run-1\n\nModel Attributed credits Est. cost\norders_with_a_very_long_model... 0.003429 $2.5000\nTOTAL 0.006858 $0.0206\nShowing 1 of 2 by estimated cost; `sqb cost run-1` for all models (1 more).\nConfigure cost.usd_per_credit with your Snowflake contract rate.\nEstimate uses fair-share visible warehouse busy time; it is not Snowflake-billed credits or invoice reconciliation.", + }, + ]; + + for test_case in test_cases { + let text: String = format_cost_breakdown( + &test_case.run, + BreakdownOptions { + use_color: false, + limit: test_case.limit, + terminal_width: test_case.terminal_width, + show_run_id: true, + }, + ); + assert_eq!(text, test_case.expected_text, "{}", test_case.description); + } +} + +#[test] +fn given_offset_timestamps_when_formatting_history_then_completion_is_shown_in_utc() { + let test_cases = [ + HistoryTestCase { + description: "a negative offset crosses midnight into the next day", + completed_at: "2026-12-31T23:30:00.5-05:00", + expected_text: "SQLBuild cost history\nCompleted (UTC) Run Build Cost Attributed credits Est. USD\n2027-01-01 04:30:00 run-1 success complete 0.006858 $0.0206\nShowing 1 of 3 matching runs. Run `sqb cost ` for a full breakdown.", + }, + HistoryTestCase { + description: "a positive offset on a leap day moves back a day", + completed_at: "2028-03-01T01:00:00+02:00", + expected_text: "SQLBuild cost history\nCompleted (UTC) Run Build Cost Attributed credits Est. USD\n2028-02-29 23:00:00 run-1 success complete 0.006858 $0.0206\nShowing 1 of 3 matching runs. Run `sqb cost ` for a full breakdown.", + }, + ]; + + for test_case in test_cases { + let text: String = + format_cost_history(&[run(test_case.completed_at, Vec::new())], 3, false); + assert_eq!(text, test_case.expected_text, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-core/src/cost_report/tests/test_types.rs b/crates/sqlbuild-core/src/cost_report/tests/test_types.rs new file mode 100644 index 0000000000..1701900259 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/tests/test_types.rs @@ -0,0 +1,15 @@ +use crate::cost_report::models::CostRun; + +pub(super) struct BreakdownTestCase { + pub(super) description: &'static str, + pub(super) run: CostRun, + pub(super) limit: Option, + pub(super) terminal_width: usize, + pub(super) expected_text: &'static str, +} + +pub(super) struct HistoryTestCase { + pub(super) description: &'static str, + pub(super) completed_at: &'static str, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/debug_report/constants.rs b/crates/sqlbuild-core/src/debug_report/constants.rs new file mode 100644 index 0000000000..320f8a6ef8 --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/constants.rs @@ -0,0 +1,21 @@ +//! `sqb debug` file names, masking and wording. + +pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; +pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml"; +pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml"; +/// Connection keys whose values are always masked. +pub(crate) const SECRET_CONNECTION_KEYS: [&str; 5] = [ + "password", + "private_key", + "private_key_file_pwd", + "secret", + "token", +]; +pub(crate) const MASKED_VALUE: &str = "****"; +/// Longer connection values are shortened to their first and last characters. +pub(crate) const PREVIEW_CHARACTER_LIMIT: usize = 32; +pub(crate) const PREVIEW_EDGE_CHARACTERS: usize = 4; +pub(crate) const NOT_CONFIGURED: &str = "not configured"; +pub(crate) const NOT_PRESENT: &str = "not present"; +pub(crate) const HEADER: &str = "SQLBuild Diagnostics"; diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs b/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs new file mode 100644 index 0000000000..6ab96b510a --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs @@ -0,0 +1,39 @@ +//! The project and local config files of a project directory. + +use std::path::{Path, PathBuf}; + +use crate::debug_report::constants::{ + LEGACY_LOCAL_CONFIG_FILENAME, LEGACY_PROJECT_CONFIG_FILENAME, LOCAL_CONFIG_FILENAME, + PROJECT_CONFIG_FILENAME, +}; +use crate::debug_report::models::DebugConfigFiles; + +fn existing_path(project_dir: &Path, preferred: &str, legacy: &str) -> PathBuf { + let preferred_path: PathBuf = project_dir.join(preferred); + if preferred_path.exists() { + return preferred_path; + } + let legacy_path: PathBuf = project_dir.join(legacy); + if legacy_path.exists() { + return legacy_path; + } + preferred_path +} + +/// The preferred config files, or their legacy YAML names when only those exist. +#[must_use] +pub fn debug_config_files(project_dir: &Path) -> DebugConfigFiles { + let local_path: PathBuf = existing_path( + project_dir, + LOCAL_CONFIG_FILENAME, + LEGACY_LOCAL_CONFIG_FILENAME, + ); + DebugConfigFiles { + project_file: existing_path( + project_dir, + PROJECT_CONFIG_FILENAME, + LEGACY_PROJECT_CONFIG_FILENAME, + ), + local_file: local_path.exists().then_some(local_path), + } +} diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_configuration_lines.rs b/crates/sqlbuild-core/src/debug_report/main/debug_configuration_lines.rs new file mode 100644 index 0000000000..4c677935c8 --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/main/debug_configuration_lines.rs @@ -0,0 +1,81 @@ +//! The configuration section of `sqb debug`. + +use crate::debug_report::constants::{NOT_CONFIGURED, NOT_PRESENT}; +use crate::debug_report::models::{DebugLine, DebugProjectFacts, DebugStatus, WarehouseCheck}; + +fn resolved(label: &str, message: &str, status_message: &str) -> DebugLine { + DebugLine::checked(label, message, (DebugStatus::Ok, Some(status_message))) +} + +fn warehouse_line(check: &WarehouseCheck) -> DebugLine { + match check { + WarehouseCheck::NotUsed { + group, + adapter_name, + source, + } => DebugLine::checked( + &format!("{group} warehouse"), + &format!("not used (adapter {adapter_name})"), + (DebugStatus::Skip, Some(source)), + ), + WarehouseCheck::Unresolved { group, error } => DebugLine::checked( + &format!("{group} warehouse"), + &format!("error: {error}"), + (DebugStatus::Error, Some("unresolved")), + ), + WarehouseCheck::Resolved { + group, + warehouse, + source, + } => resolved( + &format!("{group} warehouse"), + warehouse.as_deref().unwrap_or("not set"), + source, + ), + } +} + +/// Project file, local config, project, adapter, target, namespace and warehouse lines. +#[must_use] +pub fn debug_configuration_lines( + facts: &DebugProjectFacts, + warehouses: &[WarehouseCheck], +) -> Vec { + let local_file: Option<&str> = facts.local_file.as_deref(); + let mut lines: Vec = vec![ + resolved("project file", &facts.project_file, "found and valid"), + resolved( + "local config", + local_file.unwrap_or(NOT_PRESENT), + if local_file.is_some() { + "found" + } else { + NOT_PRESENT + }, + ), + resolved("project", &facts.project_name, "loaded"), + resolved("adapter", &facts.adapter_name, "found"), + resolved( + "target", + facts.target_name.as_deref().unwrap_or("default"), + "resolved", + ), + resolved( + "connection", + facts.connection_name.as_deref().unwrap_or("inline"), + "resolved", + ), + resolved( + "database", + facts.database.as_deref().unwrap_or(NOT_CONFIGURED), + "resolved", + ), + resolved( + "schema", + facts.schema.as_deref().unwrap_or(NOT_CONFIGURED), + "resolved", + ), + ]; + lines.extend(warehouses.iter().map(warehouse_line)); + lines +} diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_connection_lines.rs b/crates/sqlbuild-core/src/debug_report/main/debug_connection_lines.rs new file mode 100644 index 0000000000..f0e8591b0e --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/main/debug_connection_lines.rs @@ -0,0 +1,77 @@ +//! The connection section of `sqb debug`. + +use crate::debug_report::constants::{ + MASKED_VALUE, PREVIEW_CHARACTER_LIMIT, PREVIEW_EDGE_CHARACTERS, SECRET_CONNECTION_KEYS, +}; +use crate::debug_report::models::{ConnectionCheck, ConnectionValue, DebugLine, DebugStatus}; + +fn displayed_value(key: &str, value: &ConnectionValue) -> String { + if SECRET_CONNECTION_KEYS.contains(&key.to_lowercase().as_str()) { + return MASKED_VALUE.to_owned(); + } + let text: &str = match value { + ConnectionValue::Null => return "null".to_owned(), + ConnectionValue::Scalar(text) => return text.clone(), + ConnectionValue::Text(text) => text, + }; + let characters: Vec = text.chars().collect(); + if characters.len() <= PREVIEW_CHARACTER_LIMIT { + return text.to_owned(); + } + let head: String = characters[..PREVIEW_EDGE_CHARACTERS].iter().collect(); + let tail: String = characters[characters.len() - PREVIEW_EDGE_CHARACTERS..] + .iter() + .collect(); + format!("{head}...{tail}") +} + +fn check_lines(check: &ConnectionCheck) -> [DebugLine; 2] { + let line = |label: &str, status: DebugStatus, message: &str| { + DebugLine::checked(label, "", (status, Some(message))) + }; + match check { + ConnectionCheck::Skipped => [ + line( + "connection test", + DebugStatus::Skip, + "skipped by --no-connection", + ), + line("query test", DebugStatus::Skip, "connection skipped"), + ], + ConnectionCheck::ConnectFailed(error) => [ + line("connection test", DebugStatus::Error, error), + line("query test", DebugStatus::Skip, "connection failed"), + ], + ConnectionCheck::Connected { query_error } => [ + line("connection test", DebugStatus::Ok, "connected"), + match query_error { + Some(error) => line("query test", DebugStatus::Error, error), + None => line("query test", DebugStatus::Ok, "SELECT 1"), + }, + ], + } +} + +/// Connection settings sorted by key with secrets masked, then the connection checks. +#[must_use] +pub fn debug_connection_lines( + settings: &[(String, ConnectionValue)], + check: &ConnectionCheck, +) -> Vec { + let mut lines: Vec = if settings.is_empty() { + vec![DebugLine::checked( + "settings", + "no keys", + (DebugStatus::Ok, None), + )] + } else { + let mut sorted: Vec<&(String, ConnectionValue)> = settings.iter().collect(); + sorted.sort_by(|left, right| left.0.cmp(&right.0)); + sorted + .into_iter() + .map(|(key, value)| DebugLine::plain(key, &displayed_value(key, value))) + .collect() + }; + lines.extend(check_lines(check)); + lines +} diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs b/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs new file mode 100644 index 0000000000..0e411d654e --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs @@ -0,0 +1,28 @@ +//! The providers section of `sqb debug`. + +use crate::debug_report::models::{DebugLine, DebugProvider, DebugStatus}; + +/// The provider count followed by one line per provider. +#[must_use] +pub fn debug_provider_lines(providers: &[DebugProvider]) -> Vec { + if providers.is_empty() { + return vec![DebugLine::checked( + "providers", + "none discovered", + (DebugStatus::Ok, None), + )]; + } + let mut lines: Vec = vec![DebugLine::checked( + "providers", + &providers.len().to_string(), + (DebugStatus::Ok, Some("discovered")), + )]; + lines.extend(providers.iter().map(|provider| { + DebugLine::checked( + &provider.name, + &format!("{}:{}", provider.relative_path, provider.class_name), + (DebugStatus::Ok, Some("valid settings")), + ) + })); + lines +} diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_runtime_lines.rs b/crates/sqlbuild-core/src/debug_report/main/debug_runtime_lines.rs new file mode 100644 index 0000000000..7640936f1a --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/main/debug_runtime_lines.rs @@ -0,0 +1,17 @@ +//! The runtime section of `sqb debug`. + +use crate::debug_report::models::{DebugLine, DebugRuntime}; + +/// The SQLBuild version, interpreter and platform lines. +#[must_use] +pub fn debug_runtime_lines(runtime: &DebugRuntime) -> Vec { + vec![ + DebugLine::plain( + "sqlbuild version", + runtime.sqlbuild_version.as_deref().unwrap_or("unknown"), + ), + DebugLine::plain("python version", &runtime.python_version), + DebugLine::plain("python path", &runtime.python_path), + DebugLine::plain("os info", &runtime.os_info), + ] +} diff --git a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs new file mode 100644 index 0000000000..04e8d3084f --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs @@ -0,0 +1,52 @@ +//! `sqb debug --json` output. + +use crate::debug_report::models::{DebugLine, DebugReport}; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +fn line_value(line: &DebugLine) -> JsonValue { + let mut fields: Vec<(String, JsonValue)> = vec![ + ("label".to_owned(), JsonValue::String(line.label.clone())), + ( + "message".to_owned(), + JsonValue::String(line.message.clone()), + ), + ]; + if let Some(status) = line.status { + fields.push(( + "status".to_owned(), + JsonValue::String(status.label().to_owned()), + )); + } + if let Some(status_message) = &line.status_message { + fields.push(( + "status_message".to_owned(), + JsonValue::String(status_message.clone()), + )); + } + JsonValue::Object(fields) +} + +fn lines_value(lines: &[DebugLine]) -> JsonValue { + JsonValue::Array(lines.iter().map(line_value).collect()) +} + +/// `json.dumps(payload, sort_keys=True)` of the report and its success flag. +#[must_use] +pub fn format_debug_json(report: &DebugReport) -> String { + let payload: JsonValue = JsonValue::Object(vec![ + ("success".to_owned(), JsonValue::Bool(report.success())), + ("runtime".to_owned(), lines_value(&report.runtime)), + ( + "configuration".to_owned(), + lines_value(&report.configuration), + ), + ("providers".to_owned(), lines_value(&report.providers)), + ("connection".to_owned(), lines_value(&report.connection)), + ]); + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None) + }; + dumps(&payload, &JsonDialect::Stdlib(options)).unwrap_or_default() +} diff --git a/crates/sqlbuild-core/src/debug_report/main/format_debug_text.rs b/crates/sqlbuild-core/src/debug_report/main/format_debug_text.rs new file mode 100644 index 0000000000..cea1661095 --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/main/format_debug_text.rs @@ -0,0 +1,39 @@ +//! Human `sqb debug` output. + +use crate::cli_text::models::CliStyle; +use crate::debug_report::constants::HEADER; +use crate::debug_report::models::{DebugLine, DebugReport}; + +fn section(name: &str, (entries, style): (&[DebugLine], CliStyle)) -> Vec { + let mut lines: Vec = Vec::new(); + lines.push(style.section(&format!("{name}:"))); + for line in entries { + let mut message: String = line.message.clone(); + if let Some(status) = line.status { + let status_message: &str = line.status_message.as_deref().unwrap_or(&line.message); + let rendered: String = style.status_text( + status.label(), + &format!("[{} {status_message}]", status.label()), + ); + message = if message.is_empty() { + rendered + } else { + format!("{message} {rendered}") + }; + } + lines.push(format!(" {}: {message}", line.label)); + } + lines.push(String::new()); + lines +} + +/// The report as sectioned `label: message [STATUS detail]` lines. +#[must_use] +pub fn format_debug_text(report: &DebugReport, style: CliStyle) -> String { + let mut lines: Vec = vec![String::new(), style.title(HEADER), String::new()]; + lines.extend(section("Runtime", (&report.runtime, style))); + lines.extend(section("Configuration", (&report.configuration, style))); + lines.extend(section("Providers", (&report.providers, style))); + lines.extend(section("Connection", (&report.connection, style))); + lines.join("\n") + "\n" +} diff --git a/crates/sqlbuild-core/src/debug_report/main/mod.rs b/crates/sqlbuild-core/src/debug_report/main/mod.rs new file mode 100644 index 0000000000..764b8549c6 --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/main/mod.rs @@ -0,0 +1,7 @@ +pub mod debug_config_files; +pub mod debug_configuration_lines; +pub mod debug_connection_lines; +pub mod debug_provider_lines; +pub mod debug_runtime_lines; +pub mod format_debug_json; +pub mod format_debug_text; diff --git a/crates/sqlbuild-core/src/debug_report/mod.rs b/crates/sqlbuild-core/src/debug_report/mod.rs new file mode 100644 index 0000000000..17d3abdfe6 --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/debug_report/models.rs b/crates/sqlbuild-core/src/debug_report/models.rs new file mode 100644 index 0000000000..8740cbca71 --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/models.rs @@ -0,0 +1,169 @@ +//! `sqb debug` report lines and the facts they are built from. + +use std::path::PathBuf; + +/// The outcome a diagnostic line reports (`DebugCheckStatus`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DebugStatus { + Ok, + Error, + Skip, +} + +impl DebugStatus { + /// The status value shown in text and JSON output. + #[must_use] + pub fn label(self) -> &'static str { + match self { + Self::Ok => "OK", + Self::Error => "ERROR", + Self::Skip => "SKIP", + } + } +} + +/// One diagnostic line: a label, a message and an optional check status. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DebugLine { + pub label: String, + pub message: String, + pub status: Option, + pub status_message: Option, +} + +/// Every section of the `sqb debug` report. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DebugReport { + pub runtime: Vec, + pub configuration: Vec, + pub providers: Vec, + pub connection: Vec, +} + +impl DebugReport { + /// No line reports an error. + #[must_use] + pub fn success(&self) -> bool { + [ + &self.runtime, + &self.configuration, + &self.providers, + &self.connection, + ] + .into_iter() + .flatten() + .all(|line| line.status != Some(DebugStatus::Error)) + } +} + +/// The interpreter and platform SQLBuild runs on. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DebugRuntime { + pub sqlbuild_version: Option, + pub python_version: String, + pub python_path: String, + pub os_info: String, +} + +/// The project configuration files `sqb debug` reports. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DebugConfigFiles { + pub project_file: PathBuf, + /// The local config file, when it exists. + pub local_file: Option, +} + +/// The resolved project, adapter, target and namespace. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DebugProjectFacts { + /// The project config file, as Python prints the path. + pub project_file: String, + /// The local config file, when it exists. + pub local_file: Option, + pub project_name: String, + pub adapter_name: String, + pub target_name: Option, + pub connection_name: Option, + /// The effective database, when it is a non-empty string. + pub database: Option, + /// The effective schema, when it is a non-empty string. + pub schema: Option, +} + +/// One command group's warehouse as `sqb debug` resolved it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum WarehouseCheck { + /// The adapter has no session warehouse, but the target authors one at `source`. + NotUsed { + group: String, + adapter_name: String, + source: String, + }, + Unresolved { + group: String, + error: String, + }, + Resolved { + group: String, + warehouse: Option, + source: String, + }, +} + +/// One discovered provider. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DebugProvider { + pub name: String, + pub relative_path: String, + pub class_name: String, +} + +/// One connection setting value, as Python would print it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConnectionValue { + /// `None`. + Null, + /// A `bool`, `int` or `float`, as `str(value)`; never shortened. + Scalar(String), + /// Any other value, as `str(value)`. + Text(String), +} + +/// What the live connection and `SELECT 1` checks found. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConnectionCheck { + /// `--no-connection` was passed. + Skipped, + /// Connecting raised this message. + ConnectFailed(String), + /// Connected; the query raised this message when it failed. + Connected { query_error: Option }, +} + +impl DebugLine { + /// A line without a check status. + #[must_use] + pub fn plain(label: &str, message: &str) -> Self { + Self { + label: label.to_owned(), + message: message.to_owned(), + status: None, + status_message: None, + } + } + + /// A line with a check status and an optional status message. + #[must_use] + pub fn checked( + label: &str, + message: &str, + (status, status_message): (DebugStatus, Option<&str>), + ) -> Self { + Self { + label: label.to_owned(), + message: message.to_owned(), + status: Some(status), + status_message: status_message.map(str::to_owned), + } + } +} diff --git a/crates/sqlbuild-core/src/debug_report/tests.rs b/crates/sqlbuild-core/src/debug_report/tests.rs new file mode 100644 index 0000000000..d4e53679ac --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_debug_report.rs"] +mod debug_report; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs new file mode 100644 index 0000000000..623f516cd6 --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs @@ -0,0 +1,41 @@ +use crate::debug_report::models::{ConnectionValue, DebugLine, DebugReport, DebugStatus}; + +pub(super) fn sample_report() -> DebugReport { + DebugReport { + runtime: vec![DebugLine::plain("sqlbuild version", "0.2.1")], + configuration: vec![DebugLine::checked( + "project file", + "/repo/sqlbuild_project.toml", + (DebugStatus::Ok, Some("found and valid")), + )], + providers: vec![ + DebugLine::checked("providers", "1", (DebugStatus::Ok, Some("discovered"))), + DebugLine::checked( + "marker_provider", + "providers/marker.py:MarkerProvider", + (DebugStatus::Ok, Some("valid settings")), + ), + ], + connection: vec![ + DebugLine::plain("token", "****"), + DebugLine::checked( + "connection test", + "", + (DebugStatus::Error, Some("authentication failed")), + ), + DebugLine::checked( + "query test", + "", + (DebugStatus::Skip, Some("connection failed")), + ), + ], + } +} + +pub(super) fn text(value: &str) -> ConnectionValue { + ConnectionValue::Text(value.to_owned()) +} + +pub(super) fn line_summary(line: &DebugLine) -> (&str, &str, Option<&str>) { + (&line.label, &line.message, line.status_message.as_deref()) +} diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs new file mode 100644 index 0000000000..a2d2323351 --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs @@ -0,0 +1,101 @@ +use crate::cli_text::models::CliStyle; +use crate::debug_report::main::debug_connection_lines::debug_connection_lines; +use crate::debug_report::main::format_debug_json::format_debug_json; +use crate::debug_report::main::format_debug_text::format_debug_text; +use crate::debug_report::models::{ConnectionCheck, ConnectionValue, DebugLine}; +use crate::debug_report::tests::helpers::{line_summary, sample_report, text}; +use crate::debug_report::tests::test_types::{ConnectionLinesTestCase, FormatTestCase}; + +/// Expectations recorded from the retired Python formatter. +#[test] +fn given_debug_report_when_formatting_then_matches_python_outputs() { + let test_cases = [FormatTestCase { + description: "renders diagnostics sections and json sections", + report: sample_report(), + expected_text: "\nSQLBuild Diagnostics\n\nRuntime:\n sqlbuild version: 0.2.1\n\nConfiguration:\n project file: /repo/sqlbuild_project.toml [OK found and valid]\n\nProviders:\n providers: 1 [OK discovered]\n marker_provider: providers/marker.py:MarkerProvider [OK valid settings]\n\nConnection:\n token: ****\n connection test: [ERROR authentication failed]\n query test: [SKIP connection failed]\n\n", + expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, + expected_color_fragments: &[ + "\x1b[34m\x1b[1mSQLBuild Diagnostics\x1b[0m", + "\x1b[1mRuntime:\x1b[0m", + "\x1b[32m[OK found and valid]\x1b[0m", + "\x1b[38;5;167m[ERROR authentication failed]\x1b[0m", + "\x1b[2m[SKIP connection failed]\x1b[0m", + ], + }]; + + for test_case in test_cases { + let plain: String = format_debug_text(&test_case.report, CliStyle { use_color: false }); + let colored: String = format_debug_text(&test_case.report, CliStyle { use_color: true }); + assert_eq!(plain, test_case.expected_text, "{}", test_case.description); + assert_eq!( + format_debug_json(&test_case.report), + test_case.expected_json, + "{}", + test_case.description + ); + for fragment in test_case.expected_color_fragments { + assert!( + colored.contains(fragment), + "{}: {fragment:?}", + test_case.description + ); + } + } +} + +#[test] +fn given_connection_settings_when_listing_then_values_are_sorted_masked_and_shortened() { + let test_cases = [ + ConnectionLinesTestCase { + description: "secrets mask, long text shortens and scalars stay whole", + settings: vec![ + ( + "user".to_owned(), + text("a_very_long_user_name_exceeding_32_chars"), + ), + ("Password".to_owned(), text("hunter2")), + ( + "port".to_owned(), + ConnectionValue::Scalar("12345678901234567890123456789012345".to_owned()), + ), + ("role".to_owned(), ConnectionValue::Null), + ( + "account".to_owned(), + text("ünïcödé-ünïcödé-ünïcödé-ünïcödé-x"), + ), + ], + check: ConnectionCheck::Connected { + query_error: Some("boom".to_owned()), + }, + expected_lines: vec![ + ("Password", "****", None), + ("account", "ünïc...dé-x", None), + ("port", "12345678901234567890123456789012345", None), + ("role", "null", None), + ("user", "a_ve...hars", None), + ("connection test", "", Some("connected")), + ("query test", "", Some("boom")), + ], + }, + ConnectionLinesTestCase { + description: "no settings and a failed connection", + settings: Vec::new(), + check: ConnectionCheck::ConnectFailed("refused".to_owned()), + expected_lines: vec![ + ("settings", "no keys", None), + ("connection test", "", Some("refused")), + ("query test", "", Some("connection failed")), + ], + }, + ]; + + for test_case in test_cases { + let lines: Vec = debug_connection_lines(&test_case.settings, &test_case.check); + let summaries: Vec<(&str, &str, Option<&str>)> = lines.iter().map(line_summary).collect(); + assert_eq!( + summaries, test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_types.rs b/crates/sqlbuild-core/src/debug_report/tests/test_types.rs new file mode 100644 index 0000000000..4687aee553 --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/tests/test_types.rs @@ -0,0 +1,16 @@ +use crate::debug_report::models::{ConnectionCheck, ConnectionValue, DebugReport}; + +pub(super) struct FormatTestCase { + pub(super) description: &'static str, + pub(super) report: DebugReport, + pub(super) expected_text: &'static str, + pub(super) expected_json: &'static str, + pub(super) expected_color_fragments: &'static [&'static str], +} + +pub(super) struct ConnectionLinesTestCase { + pub(super) description: &'static str, + pub(super) settings: Vec<(String, ConnectionValue)>, + pub(super) check: ConnectionCheck, + pub(super) expected_lines: Vec<(&'static str, &'static str, Option<&'static str>)>, +} diff --git a/crates/sqlbuild-core/src/freshness_text/_helpers/counts.rs b/crates/sqlbuild-core/src/freshness_text/_helpers/counts.rs new file mode 100644 index 0000000000..07d7329c87 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/_helpers/counts.rs @@ -0,0 +1,21 @@ +//! Status counts of a freshness run. + +use crate::freshness_text::models::{FreshnessSourceResult, FreshnessStatus}; + +/// Sources with `status`. +pub(crate) fn status_count(results: &[FreshnessSourceResult], status: FreshnessStatus) -> i64 { + let count: usize = results + .iter() + .filter(|result| result.status == status) + .count(); + i64::try_from(count).unwrap_or(i64::MAX) +} + +/// Sources whose age policy evaluated to `age_status`. +pub(crate) fn age_count(results: &[FreshnessSourceResult], age_status: &str) -> i64 { + let count: usize = results + .iter() + .filter(|result| result.age_status.as_deref() == Some(age_status)) + .count(); + i64::try_from(count).unwrap_or(i64::MAX) +} diff --git a/crates/sqlbuild-core/src/freshness_text/_helpers/detail.rs b/crates/sqlbuild-core/src/freshness_text/_helpers/detail.rs new file mode 100644 index 0000000000..bdbc662b2d --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/_helpers/detail.rs @@ -0,0 +1,53 @@ +//! The detail text after each source name. + +use crate::cli_text::models::CliStyle; +use crate::freshness_text::models::{FreshnessSourceResult, FreshnessStatus}; + +fn muted_part(style: CliStyle, (label, value): (&str, Option<&String>)) -> Option { + value.map(|value| style.muted(&format!("{label} {value}"))) +} + +/// The detail text of one source line. +pub(crate) fn detail(result: &FreshnessSourceResult, style: CliStyle) -> String { + let mut parts: Vec = Vec::new(); + match result.status { + FreshnessStatus::Observed => { + parts.extend(muted_part( + style, + ("value", result.current_data_version.as_ref()), + )); + parts.extend(muted_part(style, ("kind", result.value_kind.as_ref()))); + parts.extend(muted_part(style, ("via", result.strategy.as_ref()))); + } + FreshnessStatus::Changed | FreshnessStatus::Unchanged | FreshnessStatus::Tolerated => { + parts.extend(muted_part( + style, + ("previous", result.previous_data_version.as_ref()), + )); + parts.extend(muted_part( + style, + ("current", result.current_data_version.as_ref()), + )); + } + FreshnessStatus::Error => { + return result + .message + .as_deref() + .map_or_else(String::new, |message| style.error(message)); + } + FreshnessStatus::Unknown => { + return result + .message + .as_deref() + .map_or_else(String::new, |message| style.warning(message)); + } + } + parts.extend(muted_part( + style, + ("tolerance", result.lag_tolerance.as_ref()), + )); + if let Some(age) = &result.age_status { + parts.push(style.status_text(age, &format!("age {age}"))); + } + parts.join(" ") +} diff --git a/crates/sqlbuild-core/src/freshness_text/_helpers/json_payload.rs b/crates/sqlbuild-core/src/freshness_text/_helpers/json_payload.rs new file mode 100644 index 0000000000..095df57600 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/_helpers/json_payload.rs @@ -0,0 +1,47 @@ +//! JSON payload pieces of freshness output. + +use crate::freshness_text::models::FreshnessSourceResult; +use crate::json::models::JsonValue; + +fn text(value: Option<&str>) -> JsonValue { + value.map_or(JsonValue::Null, |value| JsonValue::String(value.to_owned())) +} + +/// A JSON object with entries in order. +pub(crate) fn object(entries: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + entries + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ) +} + +/// The JSON payload of one source result. +pub(crate) fn source(result: &FreshnessSourceResult) -> JsonValue { + object(vec![ + ("name", text(Some(&result.name))), + ("status", text(Some(result.status.value()))), + ("strategy", text(result.strategy.as_deref())), + ("value_kind", text(result.value_kind.as_deref())), + ( + "current_data_version", + text(result.current_data_version.as_deref()), + ), + ( + "previous_data_version", + text(result.previous_data_version.as_deref()), + ), + ("lag_tolerance", text(result.lag_tolerance.as_deref())), + ( + "target", + object(vec![ + ("database", text(result.target_database.as_deref())), + ("schema", text(result.target_schema.as_deref())), + ("name", text(result.target_name.as_deref())), + ]), + ), + ("message", text(result.message.as_deref())), + ("age_status", text(result.age_status.as_deref())), + ]) +} diff --git a/crates/sqlbuild-core/src/freshness_text/_helpers/mod.rs b/crates/sqlbuild-core/src/freshness_text/_helpers/mod.rs new file mode 100644 index 0000000000..d8f1b2bb79 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod counts; +pub(crate) mod detail; +pub(crate) mod json_payload; diff --git a/crates/sqlbuild-core/src/freshness_text/constants.rs b/crates/sqlbuild-core/src/freshness_text/constants.rs new file mode 100644 index 0000000000..043fd5d680 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/constants.rs @@ -0,0 +1,34 @@ +//! Freshness output labels and glyphs. + +use crate::freshness_text::models::FreshnessStatus; + +pub(crate) const TREE_BRANCH_GLYPH: &str = "\u{251c}\u{2500}\u{2500}"; +pub(crate) const TREE_LAST_GLYPH: &str = "\u{2514}\u{2500}\u{2500}"; + +/// Status groups in output order, with their titles. +pub(crate) const GROUPS: [(FreshnessStatus, &str); 6] = [ + (FreshnessStatus::Observed, "Observed"), + (FreshnessStatus::Changed, "Changed"), + (FreshnessStatus::Unchanged, "Unchanged"), + (FreshnessStatus::Tolerated, "Tolerated"), + (FreshnessStatus::Unknown, "Unknown"), + (FreshnessStatus::Error, "Errors"), +]; + +/// Status footer labels, their JSON keys, and the status each counts. +pub(crate) const STATUS_SUMMARY: [(&str, &str, FreshnessStatus); 6] = [ + ("OBSERVED", "observed", FreshnessStatus::Observed), + ("CHANGED", "changed", FreshnessStatus::Changed), + ("UNCHANGED", "unchanged", FreshnessStatus::Unchanged), + ("TOLERATED", "tolerated", FreshnessStatus::Tolerated), + ("UNKNOWN", "unknown", FreshnessStatus::Unknown), + ("ERROR", "errors", FreshnessStatus::Error), +]; + +/// Age footer labels, their JSON keys, and the age status each counts. +pub(crate) const AGE_SUMMARY: [(&str, &str, &str); 4] = [ + ("PASS", "age_pass", "pass"), + ("WARN", "age_warn", "warn"), + ("ERROR", "age_error", "error"), + ("UNKNOWN", "age_unknown", "unknown"), +]; diff --git a/crates/sqlbuild-core/src/freshness_text/main/freshness_exit_code.rs b/crates/sqlbuild-core/src/freshness_text/main/freshness_exit_code.rs new file mode 100644 index 0000000000..4c88983a01 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/main/freshness_exit_code.rs @@ -0,0 +1,18 @@ +//! The exit code of `sqb freshness`. + +use crate::freshness_text::_helpers::counts::status_count; +use crate::freshness_text::models::{FreshnessSourceResult, FreshnessStatus}; + +/// 1 for stale sources under `--fail-on-stale` or failures under `--fail-on-error`, else 0. +#[must_use] +pub fn freshness_exit_code( + results: &[FreshnessSourceResult], + (fail_on_stale, fail_on_error): (bool, bool), +) -> i32 { + let unknown: i64 = status_count(results, FreshnessStatus::Unknown); + let errors: i64 = status_count(results, FreshnessStatus::Error); + if fail_on_stale && (status_count(results, FreshnessStatus::Changed) + unknown + errors) > 0 { + return 1; + } + i32::from(fail_on_error && unknown + errors > 0) +} diff --git a/crates/sqlbuild-core/src/freshness_text/main/freshness_json.rs b/crates/sqlbuild-core/src/freshness_text/main/freshness_json.rs new file mode 100644 index 0000000000..7ed6f994a5 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/main/freshness_json.rs @@ -0,0 +1,39 @@ +//! The JSON payload of `sqb freshness`. + +use crate::freshness_text::_helpers::counts::{age_count, status_count}; +use crate::freshness_text::_helpers::json_payload::{object, source}; +use crate::freshness_text::constants::{AGE_SUMMARY, STATUS_SUMMARY}; +use crate::freshness_text::models::FreshnessSourceResult; +use crate::json::errors::JsonEmitError; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +/// Sources and summary counts, as `json.dumps(..., indent=2)` without a trailing newline. +pub fn freshness_json(results: &[FreshnessSourceResult]) -> Result { + let mut summary: Vec<(&str, JsonValue)> = STATUS_SUMMARY + .iter() + .map(|(_, key, status)| { + ( + *key, + JsonValue::Integer(JsonInteger::from(status_count(results, *status))), + ) + }) + .collect(); + summary.extend(AGE_SUMMARY.iter().map(|(_, key, age)| { + ( + *key, + JsonValue::Integer(JsonInteger::from(age_count(results, age))), + ) + })); + let payload: JsonValue = object(vec![ + ( + "sources", + JsonValue::Array(results.iter().map(source).collect()), + ), + ("summary", object(summary)), + ]); + dumps( + &payload, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + ) +} diff --git a/crates/sqlbuild-core/src/freshness_text/main/freshness_status.rs b/crates/sqlbuild-core/src/freshness_text/main/freshness_status.rs new file mode 100644 index 0000000000..6044c0f419 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/main/freshness_status.rs @@ -0,0 +1,15 @@ +//! The status of an observed source. + +use crate::freshness_text::models::{FreshnessStatus, StateComparison}; + +/// Observed without state, unknown when state is missing, else unchanged, tolerated or changed. +#[must_use] +pub fn freshness_status(comparison: StateComparison) -> FreshnessStatus { + match comparison { + StateComparison::NotCompared => FreshnessStatus::Observed, + StateComparison::Missing => FreshnessStatus::Unknown, + StateComparison::Equivalent { same_hash: true } => FreshnessStatus::Unchanged, + StateComparison::Equivalent { same_hash: false } => FreshnessStatus::Tolerated, + StateComparison::Different => FreshnessStatus::Changed, + } +} diff --git a/crates/sqlbuild-core/src/freshness_text/main/mod.rs b/crates/sqlbuild-core/src/freshness_text/main/mod.rs new file mode 100644 index 0000000000..63129022f2 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/main/mod.rs @@ -0,0 +1,4 @@ +pub mod freshness_exit_code; +pub mod freshness_json; +pub mod freshness_status; +pub mod render_freshness_text; diff --git a/crates/sqlbuild-core/src/freshness_text/main/render_freshness_text.rs b/crates/sqlbuild-core/src/freshness_text/main/render_freshness_text.rs new file mode 100644 index 0000000000..dcaee2d4d9 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/main/render_freshness_text.rs @@ -0,0 +1,74 @@ +//! The human output of `sqb freshness`. + +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, SummaryCount}; +use crate::freshness_text::_helpers::counts::{age_count, status_count}; +use crate::freshness_text::_helpers::detail::detail; +use crate::freshness_text::constants::{ + AGE_SUMMARY, GROUPS, STATUS_SUMMARY, TREE_BRANCH_GLYPH, TREE_LAST_GLYPH, +}; +use crate::freshness_text::models::FreshnessSourceResult; + +/// Sources grouped by status, the status counts, and the age policy counts when any apply. +#[must_use] +pub fn render_freshness_text(results: &[FreshnessSourceResult], style: CliStyle) -> String { + let mut lines: Vec = vec![style.title("Source freshness")]; + let width: usize = results + .iter() + .map(|result| result.name.chars().count()) + .max() + .unwrap_or(0); + for (status, title) in GROUPS { + let group: Vec<&FreshnessSourceResult> = results + .iter() + .filter(|result| result.status == status) + .collect(); + if group.is_empty() { + continue; + } + lines.push(String::new()); + lines.push(style.section(&format!("{title} ({})", group.len()))); + for (index, result) in group.iter().enumerate() { + let connector: String = style.muted(if index + 1 == group.len() { + TREE_LAST_GLYPH + } else { + TREE_BRANCH_GLYPH + }); + let padding: String = " ".repeat(width.saturating_sub(result.name.chars().count())); + let detail: String = detail(result, style); + let suffix: String = if detail.is_empty() { + String::new() + } else { + format!(" {detail}") + }; + lines.push(format!( + "{connector} {}{suffix}", + style.object_name(&format!("{}{padding}", result.name)) + )); + } + } + lines.push(String::new()); + let counts: Vec = STATUS_SUMMARY + .iter() + .map(|(label, _, status)| SummaryCount { + label: (*label).to_owned(), + value: status_count(results, *status), + }) + .collect(); + lines.push(format_summary_footer(style, &counts, None)); + if results.iter().any(|result| result.age_status.is_some()) { + let ages: Vec = AGE_SUMMARY + .iter() + .map(|(label, _, age)| SummaryCount { + label: (*label).to_owned(), + value: age_count(results, age), + }) + .collect(); + lines.push(format!( + "{} {}", + style.section("Age policy"), + format_summary_footer(style, &ages, None) + )); + } + format!("{}\n", lines.join("\n")) +} diff --git a/crates/sqlbuild-core/src/freshness_text/mod.rs b/crates/sqlbuild-core/src/freshness_text/mod.rs new file mode 100644 index 0000000000..0246183d5c --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/mod.rs @@ -0,0 +1,9 @@ +//! `sqb freshness`: source status classification, text and JSON output, and exit codes. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/freshness_text/models.rs b/crates/sqlbuild-core/src/freshness_text/models.rs new file mode 100644 index 0000000000..48db515315 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/models.rs @@ -0,0 +1,79 @@ +//! Source freshness command results. + +/// Command status for one source freshness observation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FreshnessStatus { + Observed, + Changed, + Unchanged, + Tolerated, + Unknown, + Error, +} + +impl FreshnessStatus { + /// The status as JSON output spells it. + #[must_use] + pub fn value(self) -> &'static str { + match self { + Self::Observed => "observed", + Self::Changed => "changed", + Self::Unchanged => "unchanged", + Self::Tolerated => "tolerated", + Self::Unknown => "unknown", + Self::Error => "error", + } + } +} + +/// Freshness observation result for one source. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FreshnessSourceResult { + pub name: String, + pub status: FreshnessStatus, + pub strategy: Option, + pub value_kind: Option, + pub current_data_version: Option, + pub previous_data_version: Option, + pub lag_tolerance: Option, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub message: Option, + /// `pass`, `warn`, `error` or `unknown` when the source has an age policy. + pub age_status: Option, +} + +impl FreshnessSourceResult { + /// A result with only a name and status set. + #[must_use] + pub fn new(name: &str, status: FreshnessStatus) -> Self { + Self { + name: name.to_owned(), + status, + strategy: None, + value_kind: None, + current_data_version: None, + previous_data_version: None, + lag_tolerance: None, + target_database: None, + target_schema: None, + target_name: None, + message: None, + age_status: None, + } + } +} + +/// How a current observation compares to the previous state record. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StateComparison { + /// State was not requested. + NotCompared, + /// State was requested and has no record for the source. + Missing, + /// The records are equivalent within tolerance, and whether their hashes are equal. + Equivalent { same_hash: bool }, + /// The records differ beyond tolerance. + Different, +} diff --git a/crates/sqlbuild-core/src/freshness_text/tests.rs b/crates/sqlbuild-core/src/freshness_text/tests.rs new file mode 100644 index 0000000000..f08f850618 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_freshness_text.rs"] +mod freshness_text; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/freshness_text/tests/helpers.rs b/crates/sqlbuild-core/src/freshness_text/tests/helpers.rs new file mode 100644 index 0000000000..ea3b36bcea --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/tests/helpers.rs @@ -0,0 +1,59 @@ +use crate::freshness_text::models::{FreshnessSourceResult, FreshnessStatus}; + +/// An observed timestamp source with an age status. +pub(super) fn observed( + name: &str, + version: &str, + age_status: Option<&str>, +) -> FreshnessSourceResult { + FreshnessSourceResult { + strategy: Some("sql".to_owned()), + value_kind: Some("timestamp".to_owned()), + current_data_version: Some(version.to_owned()), + age_status: age_status.map(str::to_owned), + ..FreshnessSourceResult::new(name, FreshnessStatus::Observed) + } +} + +/// A source with a status and message only. +pub(super) fn with_message( + name: &str, + status: FreshnessStatus, + message: &str, +) -> FreshnessSourceResult { + FreshnessSourceResult { + message: Some(message.to_owned()), + ..FreshnessSourceResult::new(name, status) + } +} + +/// Observed, unknown and error sources with every age status. +pub(super) fn mixed_results() -> Vec { + vec![ + FreshnessSourceResult { + lag_tolerance: Some("15m".to_owned()), + ..observed("raw_orders", "2026-01-01T00:00:00", Some("warn")) + }, + observed("raw_clicks", "2026-01-01T00:30:00", Some("pass")), + observed("raw_shipments", "2025-12-31T20:00:00", Some("error")), + observed("raw_inventory", "42", Some("unknown")), + with_message( + "raw_payments", + FreshnessStatus::Unknown, + "no freshness config and adapter metadata unavailable", + ), + with_message( + "raw_events", + FreshnessStatus::Error, + "freshness query failed", + ), + ] +} + +/// Results with one source per status. +pub(super) fn results_with(statuses: &[FreshnessStatus]) -> Vec { + statuses + .iter() + .map(|status| FreshnessSourceResult::new("raw_orders", *status)) + .collect() +} diff --git a/crates/sqlbuild-core/src/freshness_text/tests/test_freshness_text.rs b/crates/sqlbuild-core/src/freshness_text/tests/test_freshness_text.rs new file mode 100644 index 0000000000..aa0f47a3e5 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/tests/test_freshness_text.rs @@ -0,0 +1,187 @@ +use crate::cli_text::models::CliStyle; +use crate::freshness_text::main::freshness_exit_code::freshness_exit_code; +use crate::freshness_text::main::freshness_json::freshness_json; +use crate::freshness_text::main::freshness_status::freshness_status; +use crate::freshness_text::main::render_freshness_text::render_freshness_text; +use crate::freshness_text::models::{FreshnessSourceResult, FreshnessStatus, StateComparison}; +use crate::freshness_text::tests::helpers::{mixed_results, observed, results_with}; +use crate::freshness_text::tests::test_types::{ + ExitCodeTestCase, JsonTestCase, StatusTestCase, TextTestCase, +}; + +#[test] +fn given_results_when_rendering_text_then_groups_sources_by_status() { + let test_cases = [ + TextTestCase { + description: "observed, unknown and error groups with an age footer", + results: mixed_results(), + use_color: false, + expected_fragments: &[ + "Source freshness\n\nObserved (4)\n", + "\u{251c}\u{2500}\u{2500} raw_orders value 2026-01-01T00:00:00 kind timestamp via sql tolerance 15m age warn\n", + "\u{2514}\u{2500}\u{2500} raw_inventory value 42 kind timestamp via sql age unknown\n", + "\nUnknown (1)\n\u{2514}\u{2500}\u{2500} raw_payments no freshness config and adapter metadata unavailable\n", + "\nErrors (1)\n\u{2514}\u{2500}\u{2500} raw_events freshness query failed\n", + "\nOBSERVED=4 CHANGED=0 UNCHANGED=0 TOLERATED=0 UNKNOWN=1 ERROR=1\n", + "Age policy PASS=1 WARN=1 ERROR=1 UNKNOWN=1\n", + ], + }, + TextTestCase { + description: "colored output styles ages and messages by status", + results: mixed_results(), + use_color: true, + expected_fragments: &[ + "\u{1b}[34m\u{1b}[1mSource freshness\u{1b}[0m", + "\u{1b}[33mage warn\u{1b}[0m", + "\u{1b}[32mage pass\u{1b}[0m", + "\u{1b}[38;5;167mage error\u{1b}[0m", + "\u{1b}[33mno freshness config and adapter metadata unavailable\u{1b}[0m", + "\u{1b}[38;5;167mfreshness query failed\u{1b}[0m", + "\u{1b}[2mUNKNOWN=\u{1b}[0m\u{1b}[33m1\u{1b}[0m", + ], + }, + TextTestCase { + description: "compared sources show previous and current versions", + results: vec![FreshnessSourceResult { + status: FreshnessStatus::Changed, + previous_data_version: Some("1".to_owned()), + ..observed("raw_orders", "2", None) + }], + use_color: false, + expected_fragments: &[ + "Changed (1)\n\u{2514}\u{2500}\u{2500} raw_orders previous 1 current 2\n\n", + "OBSERVED=0 CHANGED=1", + ], + }, + TextTestCase { + description: "no sources render only the title and footer", + results: Vec::new(), + use_color: false, + expected_fragments: &[ + "Source freshness\n\nOBSERVED=0 CHANGED=0 UNCHANGED=0 TOLERATED=0 UNKNOWN=0 ERROR=0\n", + ], + }, + ]; + + for test_case in test_cases { + let text: String = render_freshness_text( + &test_case.results, + CliStyle { + use_color: test_case.use_color, + }, + ); + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| text.contains(fragment)), + "{}: {text}", + test_case.description + ); + } +} + +#[test] +fn given_results_when_serializing_then_matches_indented_json() { + let test_cases = [JsonTestCase { + description: "one observed source with all fields and the summary", + results: vec![observed("raw_orders", "1", Some("pass"))], + expected_json: "{\n \"sources\": [\n {\n \"name\": \"raw_orders\",\n \ + \"status\": \"observed\",\n \"strategy\": \"sql\",\n \ + \"value_kind\": \"timestamp\",\n \"current_data_version\": \"1\",\n \ + \"previous_data_version\": null,\n \"lag_tolerance\": null,\n \ + \"target\": {\n \"database\": null,\n \"schema\": null,\n \ + \"name\": null\n },\n \"message\": null,\n \"age_status\": \"pass\"\n \ + }\n ],\n \"summary\": {\n \"observed\": 1,\n \"changed\": 0,\n \ + \"unchanged\": 0,\n \"tolerated\": 0,\n \"unknown\": 0,\n \"errors\": 0,\n \ + \"age_pass\": 1,\n \"age_warn\": 0,\n \"age_error\": 0,\n \"age_unknown\": 0\n }\n}", + }]; + + for test_case in test_cases { + assert_eq!( + freshness_json(&test_case.results).ok().as_deref(), + Some(test_case.expected_json), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_statuses_when_computing_exit_code_then_honours_fail_flags() { + let test_cases = [ + ExitCodeTestCase { + description: "no flags always succeed", + statuses: &[FreshnessStatus::Error], + flags: (false, false), + expected_code: 0, + }, + ExitCodeTestCase { + description: "fail on stale fails for changed sources", + statuses: &[FreshnessStatus::Changed], + flags: (true, false), + expected_code: 1, + }, + ExitCodeTestCase { + description: "fail on error ignores changed sources", + statuses: &[FreshnessStatus::Changed, FreshnessStatus::Tolerated], + flags: (false, true), + expected_code: 0, + }, + ExitCodeTestCase { + description: "fail on error fails for unknown sources", + statuses: &[FreshnessStatus::Unknown], + flags: (false, true), + expected_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + freshness_exit_code(&results_with(test_case.statuses), test_case.flags), + test_case.expected_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_state_comparison_when_classifying_then_returns_status() { + let test_cases = [ + StatusTestCase { + description: "no state comparison observes", + comparison: StateComparison::NotCompared, + expected_status: FreshnessStatus::Observed, + }, + StatusTestCase { + description: "missing previous state is unknown", + comparison: StateComparison::Missing, + expected_status: FreshnessStatus::Unknown, + }, + StatusTestCase { + description: "equal hashes are unchanged", + comparison: StateComparison::Equivalent { same_hash: true }, + expected_status: FreshnessStatus::Unchanged, + }, + StatusTestCase { + description: "equivalent within tolerance is tolerated", + comparison: StateComparison::Equivalent { same_hash: false }, + expected_status: FreshnessStatus::Tolerated, + }, + StatusTestCase { + description: "different records changed", + comparison: StateComparison::Different, + expected_status: FreshnessStatus::Changed, + }, + ]; + + for test_case in test_cases { + assert_eq!( + freshness_status(test_case.comparison), + test_case.expected_status, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/freshness_text/tests/test_types.rs b/crates/sqlbuild-core/src/freshness_text/tests/test_types.rs new file mode 100644 index 0000000000..08610ae312 --- /dev/null +++ b/crates/sqlbuild-core/src/freshness_text/tests/test_types.rs @@ -0,0 +1,27 @@ +use crate::freshness_text::models::{FreshnessSourceResult, FreshnessStatus, StateComparison}; + +pub(super) struct TextTestCase { + pub(super) description: &'static str, + pub(super) results: Vec, + pub(super) use_color: bool, + pub(super) expected_fragments: &'static [&'static str], +} + +pub(super) struct JsonTestCase { + pub(super) description: &'static str, + pub(super) results: Vec, + pub(super) expected_json: &'static str, +} + +pub(super) struct ExitCodeTestCase { + pub(super) description: &'static str, + pub(super) statuses: &'static [FreshnessStatus], + pub(super) flags: (bool, bool), + pub(super) expected_code: i32, +} + +pub(super) struct StatusTestCase { + pub(super) description: &'static str, + pub(super) comparison: StateComparison, + pub(super) expected_status: FreshnessStatus, +} diff --git a/crates/sqlbuild-core/src/integration_results/_helpers/bounded.rs b/crates/sqlbuild-core/src/integration_results/_helpers/bounded.rs new file mode 100644 index 0000000000..c09ffdaeb6 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/_helpers/bounded.rs @@ -0,0 +1,192 @@ +//! Size, depth and key bounds of integration values and the allowlisted metadata mappings. + +use crate::integration_results::_helpers::values::{field, one_of, text_length}; +use crate::integration_results::constants::{ + MAX_COLLECTION_ITEMS, MAX_IDENTIFIER_CHARS, MAX_METADATA_BYTES, MAX_NESTING_DEPTH, + MAX_RECORD_BYTES, MAX_STRING_CHARS, MAXIMUM_START_ACTIONS, MAXIMUM_START_INPUT_KEYS, + MAXIMUM_START_KEYS, MAXIMUM_START_NULLABLE_KEY, MAXIMUM_START_REQUIRED_KEYS, + MAXIMUM_START_STRUCTURED_KEYS, +}; +use crate::integration_results::errors::IntegrationRecordError; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// `validate_json_value`: reject non-finite, deep, oversized or badly keyed values. +pub(crate) fn validate_json_value( + value: &JsonValue, + depth: usize, +) -> Result<(), IntegrationRecordError> { + if depth > MAX_NESTING_DEPTH { + return Err(IntegrationRecordError::new( + "integration JSON exceeds nesting limit", + )); + } + match value { + JsonValue::Null | JsonValue::Bool(_) | JsonValue::Integer(_) => Ok(()), + JsonValue::Float(number) if !number.is_finite() => Err(IntegrationRecordError::new( + "integration JSON numbers must be finite", + )), + JsonValue::Float(_) => Ok(()), + JsonValue::String(_) | JsonValue::Utf16(_) => { + if text_length(value).unwrap_or(0) > MAX_STRING_CHARS { + return Err(IntegrationRecordError::new( + "integration JSON string exceeds size limit", + )); + } + Ok(()) + } + JsonValue::Object(entries) => { + if entries.len() > MAX_COLLECTION_ITEMS { + return Err(IntegrationRecordError::new( + "integration JSON object exceeds item limit", + )); + } + for (key, item) in entries { + let length: usize = key.chars().count(); + if length == 0 || length > MAX_IDENTIFIER_CHARS { + return Err(IntegrationRecordError::new( + "integration JSON object key is invalid", + )); + } + validate_json_value(item, depth + 1)?; + } + Ok(()) + } + JsonValue::Array(items) => { + if items.len() > MAX_COLLECTION_ITEMS { + return Err(IntegrationRecordError::new( + "integration JSON array exceeds item limit", + )); + } + items + .iter() + .try_for_each(|item| validate_json_value(item, depth + 1)) + } + } +} + +/// `encode_integration_json`: strict compact ASCII JSON within the metadata or record limit. +pub(crate) fn encode(value: &JsonValue, record: bool) -> Result { + validate_json_value(value, 0)?; + let options: StdlibJsonOptions = StdlibJsonOptions { + allow_nan: false, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + let Ok(encoded) = dumps(value, &JsonDialect::Stdlib(options)) else { + return Err(IntegrationRecordError::new( + "integration value is not strict JSON", + )); + }; + let limit: usize = if record { + MAX_RECORD_BYTES + } else { + MAX_METADATA_BYTES + }; + if encoded.len() > limit { + return Err(IntegrationRecordError::new( + "integration JSON exceeds size limit", + )); + } + Ok(encoded) +} + +fn nested_keys_allowed(value: &JsonValue, allowed: &[&str]) -> bool { + match value { + JsonValue::Object(entries) => entries.iter().all(|(key, item)| { + allowed.contains(&key.as_str()) && nested_keys_allowed(item, allowed) + }), + JsonValue::Array(items) => items.iter().all(|item| nested_keys_allowed(item, allowed)), + _ => true, + } +} + +/// `validate_allowlisted_mapping`: an object whose keys, at every depth, are allowlisted. +pub(crate) fn allowlisted_mapping( + value: &JsonValue, + field_name: &str, + allowed: &[&str], +) -> Result<(), IntegrationRecordError> { + let JsonValue::Object(entries) = value else { + return Err(IntegrationRecordError::new(format!( + "integration {field_name} must be an object" + ))); + }; + if entries + .iter() + .any(|(key, _)| !allowed.contains(&key.as_str())) + { + return Err(IntegrationRecordError::new(format!( + "integration {field_name} contains unknown fields" + ))); + } + let _ = encode(value, false)?; + if !nested_keys_allowed(value, allowed) { + return Err(IntegrationRecordError::new(format!( + "integration {field_name} contains unknown fields" + ))); + } + Ok(()) +} + +/// Whether the object has exactly the given keys. +pub(crate) fn has_exactly(entries: &[(String, JsonValue)], keys: &[&str]) -> bool { + entries.len() == keys.len() && keys.iter().all(|key| has_key(entries, key)) +} + +/// Whether the object has the key. +pub(crate) fn has_key(entries: &[(String, JsonValue)], key: &str) -> bool { + entries.iter().any(|(name, _)| name == key) +} + +/// `validate_maximum_start_safety`: empty, or the canonical maximum-start structure. +pub(crate) fn maximum_start_safety(value: &JsonValue) -> Result<(), IntegrationRecordError> { + allowlisted_mapping(value, "maximum_start_safety", &MAXIMUM_START_KEYS)?; + let JsonValue::Object(entries) = value else { + return Ok(()); + }; + if entries.is_empty() { + return Ok(()); + } + if !has_exactly(entries, &MAXIMUM_START_REQUIRED_KEYS) { + return Err(IntegrationRecordError::new( + "integration maximum_start_safety must use the canonical structure", + )); + } + if !one_of(field(entries, "action"), &MAXIMUM_START_ACTIONS) { + return Err(IntegrationRecordError::new( + "integration maximum_start_safety action is unsupported", + )); + } + for name in MAXIMUM_START_REQUIRED_KEYS { + if MAXIMUM_START_STRUCTURED_KEYS.contains(&name) { + continue; + } + let item: &JsonValue = field(entries, name); + let nullable: bool = name == MAXIMUM_START_NULLABLE_KEY && matches!(item, JsonValue::Null); + if text_length(item).is_none() && !nullable { + return Err(IntegrationRecordError::new(format!( + "integration maximum_start_safety {name} has an invalid type" + ))); + } + } + let input: &JsonValue = field(entries, "input"); + let JsonValue::Object(input) = input else { + return Err(IntegrationRecordError::new( + "integration maximum_start_safety input must use the canonical structure", + )); + }; + if !has_exactly(input, &MAXIMUM_START_INPUT_KEYS) { + return Err(IntegrationRecordError::new( + "integration maximum_start_safety input must use the canonical structure", + )); + } + if input + .iter() + .any(|(_, item)| text_length(item).is_none_or(|length| length == 0)) + { + return Err(IntegrationRecordError::new( + "integration maximum_start_safety input values must be non-empty strings", + )); + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/integration_results/_helpers/decoding.rs b/crates/sqlbuild-core/src/integration_results/_helpers/decoding.rs new file mode 100644 index 0000000000..7c3c09c97e --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/_helpers/decoding.rs @@ -0,0 +1,131 @@ +//! The phases of decoding one record: identity, its parts, and its checks. + +use crate::integration_results::_helpers::bounded::has_key; +use crate::integration_results::_helpers::records::{asset, check, invalid_envelope}; +use crate::integration_results::_helpers::values::{field, one_of}; +use crate::integration_results::constants::{ + ASSET_OUTPUT, AUDIT_SEVERITIES, CHECK_OUTPUT, MAX_COLLECTION_ITEMS, PROJECTION_DEGRADED_FIELD, + RECORD_KIND, SCHEMA_VERSION, +}; +use crate::integration_results::errors::IntegrationRecordError; +use crate::json::models::{JsonInteger, JsonValue}; + +type Entries = Vec<(String, JsonValue)>; + +/// The record's envelope fields, its asset and its checks. +pub(crate) struct RecordParts { + pub(crate) envelope: Entries, + pub(crate) asset: JsonValue, + pub(crate) checks: Vec, +} + +/// A value as Python's `str()` shows it in an error message. +fn shown(value: &JsonValue) -> String { + match value { + JsonValue::Null => "None".to_owned(), + JsonValue::String(text) => text.clone(), + JsonValue::Integer(number) => number.as_str().to_owned(), + JsonValue::Bool(flag) => if *flag { "True" } else { "False" }.to_owned(), + other => format!("{other:?}"), + } +} + +/// The record's schema version and kind are the supported ones. +pub(crate) fn require_supported( + entries: &[(String, JsonValue)], +) -> Result<(), IntegrationRecordError> { + let version: &JsonValue = field(entries, "schema_version"); + if *version != JsonValue::Integer(JsonInteger::from(SCHEMA_VERSION)) { + return Err(IntegrationRecordError::new(format!( + "unsupported integration result schema_version: {}", + shown(version) + ))); + } + let kind: &JsonValue = field(entries, "record_kind"); + if !one_of(kind, &[RECORD_KIND]) { + return Err(IntegrationRecordError::new(format!( + "unsupported integration result record_kind: {}", + shown(kind) + ))); + } + Ok(()) +} + +/// The record split into its envelope fields, raw asset and raw checks. +pub(crate) fn split(entries: Entries) -> Result { + let mut parts: RecordParts = RecordParts { + envelope: Vec::with_capacity(entries.len()), + asset: JsonValue::Null, + checks: Vec::new(), + }; + for (key, value) in entries { + match (key.as_str(), value) { + ("asset", value @ (JsonValue::Null | JsonValue::Object(_))) => parts.asset = value, + ("checks", JsonValue::Array(items)) => parts.checks = items, + ("asset", _) => { + return Err(IntegrationRecordError::new( + "integration result asset must be an object", + )); + } + ("checks", _) => { + return Err(IntegrationRecordError::new( + "integration result checks must be an array", + )); + } + (_, value) => parts.envelope.push((key, value)), + } + } + if parts.checks.len() > MAX_COLLECTION_ITEMS { + return Err(IntegrationRecordError::new( + "integration result checks exceed item limit", + )); + } + Ok(parts) +} + +/// The record's envelope fields with the default `projection_degraded`. +pub(crate) fn with_degraded_default( + mut envelope: Entries, +) -> Result { + if !one_of( + field(&envelope, "output_kind"), + &[ASSET_OUTPUT, CHECK_OUTPUT], + ) { + return Err(invalid_envelope()); + } + if !has_key(&envelope, PROJECTION_DEGRADED_FIELD) { + envelope.push((PROJECTION_DEGRADED_FIELD.to_owned(), JsonValue::Bool(false))); + } + Ok(envelope) +} + +/// `IntegrationAssetResult(**asset)` of a raw asset, or `null`. +pub(crate) fn decoded_asset(raw: JsonValue) -> Result { + match raw { + JsonValue::Object(object) => Ok(JsonValue::Object(asset(object)?)), + _ => Ok(JsonValue::Null), + } +} + +/// `IntegrationCheckResult(**check)` of every raw check. +pub(crate) fn decoded_checks( + raw: Vec, +) -> Result, IntegrationRecordError> { + let mut decoded: Vec = Vec::with_capacity(raw.len()); + for item in raw { + let JsonValue::Object(entries) = item else { + return Err(IntegrationRecordError::new( + "integration result checks must contain only objects", + )); + }; + let severity: &JsonValue = field(&entries, "severity"); + if !matches!(severity, JsonValue::Null) && !one_of(severity, &AUDIT_SEVERITIES) { + return Err(IntegrationRecordError::new(format!( + "integration check severity must be one of: {}", + AUDIT_SEVERITIES.join(", ") + ))); + } + decoded.push(JsonValue::Object(check(entries)?)); + } + Ok(decoded) +} diff --git a/crates/sqlbuild-core/src/integration_results/_helpers/enrichment.rs b/crates/sqlbuild-core/src/integration_results/_helpers/enrichment.rs new file mode 100644 index 0000000000..69c4960d91 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/_helpers/enrichment.rs @@ -0,0 +1,62 @@ +//! Optional enrichment removed from the parts of a record. + +use crate::integration_results::constants::{ + MICROBATCH_KEYS, OPTIONAL_ASSET_ENRICHMENT, OPTIONAL_CHECK_ENRICHMENT, + OPTIONAL_MICROBATCH_ENUMS, +}; +use crate::json::models::JsonValue; + +type Entries = Vec<(String, JsonValue)>; + +fn without(entries: Entries, dropped: &[&str]) -> Entries { + entries + .into_iter() + .filter(|(key, _)| !dropped.contains(&key.as_str())) + .collect() +} + +/// Whether a microbatch key survives: known, and not an optional enumeration. +fn core_microbatch_key(key: &str) -> bool { + MICROBATCH_KEYS.contains(&key) && !OPTIONAL_MICROBATCH_ENUMS.contains(&key) +} + +fn stripped_microbatch(microbatch: Entries) -> Entries { + microbatch + .into_iter() + .filter(|(key, _)| core_microbatch_key(key)) + .collect() +} + +fn stripped_asset_field((name, item): (String, JsonValue)) -> (String, JsonValue) { + match (name.as_str(), item) { + ("microbatch", JsonValue::Object(microbatch)) => { + (name, JsonValue::Object(stripped_microbatch(microbatch))) + } + (_, item) => (name, item), + } +} + +fn stripped_asset(asset: Entries) -> Entries { + without(asset, &OPTIONAL_ASSET_ENRICHMENT) + .into_iter() + .map(stripped_asset_field) + .collect() +} + +fn stripped_check(check: JsonValue) -> JsonValue { + match check { + JsonValue::Object(check) => JsonValue::Object(without(check, &OPTIONAL_CHECK_ENRICHMENT)), + other => other, + } +} + +pub(crate) fn stripped_field((key, value): (String, JsonValue)) -> (String, JsonValue) { + match (key.as_str(), value) { + ("asset", JsonValue::Object(asset)) => (key, JsonValue::Object(stripped_asset(asset))), + ("checks", JsonValue::Array(checks)) => ( + key, + JsonValue::Array(checks.into_iter().map(stripped_check).collect()), + ), + (_, value) => (key, value), + } +} diff --git a/crates/sqlbuild-core/src/integration_results/_helpers/mod.rs b/crates/sqlbuild-core/src/integration_results/_helpers/mod.rs new file mode 100644 index 0000000000..871c9b54b3 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod bounded; +pub(crate) mod decoding; +pub(crate) mod enrichment; +pub(crate) mod records; +pub(crate) mod timestamps; +pub(crate) mod values; diff --git a/crates/sqlbuild-core/src/integration_results/_helpers/records.rs b/crates/sqlbuild-core/src/integration_results/_helpers/records.rs new file mode 100644 index 0000000000..7aad864c9e --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/_helpers/records.rs @@ -0,0 +1,458 @@ +//! The asset, check and envelope parts of one record, completed and checked against the contract. + +use crate::integration_results::_helpers::bounded::{ + allowlisted_mapping, encode, has_exactly, maximum_start_safety, +}; +use crate::integration_results::_helpers::timestamps::aware_timestamp; +use crate::integration_results::_helpers::values::{ + field, finite_float, non_negative_integer, null_or_one_of, one_of, optional_identifier, + require_identifier, text, text_length, +}; +use crate::integration_results::constants::{ + ASSET_FIELDS, ASSET_KINDS, ASSET_MAPPING_FIELDS, ASSET_OUTPUT, ASSET_STATUSES, AUDIT_KIND, + AUDIT_SEVERITIES, BOUND_OPERATORS, CHECK_ATTACHMENT_KINDS, CHECK_FIELDS, CHECK_KINDS, + CHECK_NON_FAIL_STATUSES, CHECK_OUTPUT, CHECK_RUN_SCOPE_PHASES, CHECK_STATUSES, CLONE_ACTIONS, + ENVELOPE_FIELDS, EVALUATION_MODES, EXTRA_RESOURCE_KINDS, FAILED_PHASES, FAILED_STATUS, + FUTURE_CURSOR_KEYS, LOADER_KIND, MICROBATCH_COUNT_KEYS, MICROBATCH_KEYS, + MICROBATCH_LIMIT_ACTIONS, MICROBATCH_PARTITION_POLICIES, MICROBATCH_REPLAY_STATES, + MICROBATCH_RUN_TYPES, MIN_DURATION_MS, MODEL_RESOURCE_KINDS, NON_IDENTIFIER_IDENTITY_KEYS, + RANGE_OPERATOR, RESOURCE_FAILED_EVENT, RESOURCE_SKIP_CODES, RESOURCE_SKIPPED_EVENT, + RESOURCE_TERMINALS, SKIP_MODES, SKIPPED_STATUS, THRESHOLD_LEVELS, +}; +use crate::integration_results::errors::IntegrationRecordError; +use crate::json::models::JsonValue; + +/// A constructor received an unknown or missing field. +pub(crate) fn invalid_envelope() -> IntegrationRecordError { + IntegrationRecordError::new("invalid integration result envelope") +} + +fn reject(message: impl Into) -> Result<(), IntegrationRecordError> { + Err(IntegrationRecordError::new(message)) +} + +fn is_field(fields: &[(&str, bool)], key: &str) -> bool { + fields.iter().any(|(name, _)| *name == key) +} + +/// The object's fields in declaration order, defaults filled in; unknown or missing fields are invalid. +pub(crate) fn completed( + object: Vec<(String, JsonValue)>, + fields: &[(&str, bool)], + mappings: &[&str], +) -> Result, IntegrationRecordError> { + if object.iter().any(|(key, _)| !is_field(fields, key)) { + return Err(invalid_envelope()); + } + let mut ordered: Vec<(String, JsonValue)> = Vec::with_capacity(fields.len()); + for (name, required) in fields { + let value: Option = object + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.clone()); + let value: JsonValue = match value { + Some(value) => value, + None if *required => return Err(invalid_envelope()), + None if mappings.contains(name) => JsonValue::Object(Vec::new()), + None => JsonValue::Null, + }; + ordered.push(((*name).to_owned(), value)); + } + Ok(ordered) +} + +fn microbatch(value: &JsonValue) -> Result<(), IntegrationRecordError> { + allowlisted_mapping(value, "microbatch", &MICROBATCH_KEYS)?; + let JsonValue::Object(entries) = value else { + return Ok(()); + }; + for key in MICROBATCH_COUNT_KEYS { + let count: &JsonValue = field(entries, key); + if !matches!(count, JsonValue::Null) && !non_negative_integer(count) { + return reject(format!( + "integration microbatch {key} must be non-negative integer" + )); + } + } + for (key, allowed) in [ + ("run_type", &MICROBATCH_RUN_TYPES[..]), + ("action", &MICROBATCH_LIMIT_ACTIONS[..]), + ( + "unaccounted_partition_policy", + &MICROBATCH_PARTITION_POLICIES[..], + ), + ("replay_requirement_state", &MICROBATCH_REPLAY_STATES[..]), + ] { + if !null_or_one_of(field(entries, key), allowed) { + return reject(format!("integration microbatch {key} is unsupported")); + } + } + if !matches!( + field(entries, "concurrent_enabled"), + JsonValue::Null | JsonValue::Bool(_) + ) { + return reject("integration microbatch concurrent_enabled must be boolean"); + } + Ok(()) +} + +/// `IntegrationAssetResult(**asset)`: the completed asset, or the contract it breaks. +pub(crate) fn asset( + object: Vec<(String, JsonValue)>, +) -> Result, IntegrationRecordError> { + let asset: Vec<(String, JsonValue)> = completed(object, &ASSET_FIELDS, &ASSET_MAPPING_FIELDS)?; + let value = |key: &str| field(&asset, key); + require_identifier(value("kind"), "asset kind")?; + require_identifier(value("name"), "asset name")?; + if !one_of(value("kind"), &ASSET_KINDS) { + reject("integration asset kind is unsupported")?; + } + if !one_of(value("status"), &ASSET_STATUSES) { + reject("integration asset status is unsupported")?; + } + if !null_or_one_of(value("action"), &CLONE_ACTIONS) { + reject("integration asset action is unsupported")?; + } + if !null_or_one_of(value("failed_phase"), &FAILED_PHASES) { + reject("integration asset failed_phase is unsupported")?; + } + for name in [ + "target", + "origin_relation", + "staging_relation", + "failed_phase", + "loader", + ] { + optional_identifier(value(name), name)?; + } + let rows_loaded: &JsonValue = value("rows_loaded"); + if !matches!(rows_loaded, JsonValue::Null) && !non_negative_integer(rows_loaded) { + reject("integration rows_loaded must be non-negative")?; + } + if !matches!(value("materialized"), JsonValue::Null | JsonValue::Bool(_)) { + reject("integration materialized must be boolean")?; + } + allowlisted_mapping( + value("future_cursor_safety"), + "future_cursor_safety", + &FUTURE_CURSOR_KEYS, + )?; + allowlisted_mapping(value("microbatch"), "microbatch", &MICROBATCH_KEYS)?; + maximum_start_safety(value("maximum_start_safety"))?; + microbatch(value("microbatch"))?; + Ok(asset) +} + +fn thresholds(value: &JsonValue) -> Result<(), IntegrationRecordError> { + let JsonValue::Object(entries) = value else { + return reject("integration check thresholds are invalid"); + }; + if entries.is_empty() + || entries + .iter() + .any(|(key, _)| !THRESHOLD_LEVELS.contains(&key.as_str())) + { + return reject("integration check thresholds are invalid"); + } + for (_, bound) in entries { + let JsonValue::Object(bound) = bound else { + return reject("integration check threshold must be an object"); + }; + let operator: &JsonValue = field(bound, "operator"); + let expected: &[&str] = if one_of(operator, &BOUND_OPERATORS) { + &["operator", "limit"] + } else if one_of(operator, &[RANGE_OPERATOR]) { + &["operator", "lower", "upper"] + } else { + return reject("integration check threshold operator is unsupported"); + }; + if !has_exactly(bound, expected) + || expected[1..] + .iter() + .any(|key| !finite_float(field(bound, key))) + { + return reject("integration check threshold bound is invalid"); + } + } + encode(value, false).map(|_| ()) +} + +/// `IntegrationCheckResult(**check)`: the completed check, or the contract it breaks. +pub(crate) fn check( + object: Vec<(String, JsonValue)>, +) -> Result, IntegrationRecordError> { + let check: Vec<(String, JsonValue)> = completed(object, &CHECK_FIELDS, &[])?; + let value = |key: &str| field(&check, key); + require_identifier(value("kind"), "check kind")?; + require_identifier(value("name"), "check name")?; + require_identifier(value("check_id"), "check id")?; + optional_identifier(value("dag_check_id"), "dag_check_id")?; + if !one_of(value("kind"), &CHECK_KINDS) { + reject("integration check kind is unsupported")?; + } + if text(value("kind")) == Some(AUDIT_KIND) + && matches!(value("audit_definition_name"), JsonValue::Null) + { + reject("integration audit check requires audit_definition_name")?; + } + if !one_of(value("status"), &CHECK_STATUSES) { + reject("integration check status is unsupported")?; + } + let JsonValue::Bool(passed) = value("passed") else { + return Err(IntegrationRecordError::new( + "integration check passed must be boolean", + )); + }; + if *passed != one_of(value("status"), &CHECK_NON_FAIL_STATUSES) { + reject("integration check status and passed are incoherent")?; + } + if !null_or_one_of(value("severity"), &AUDIT_SEVERITIES) { + reject("integration check severity is unsupported")?; + } + if !null_or_one_of(value("attachment_kind"), &CHECK_ATTACHMENT_KINDS) { + reject("integration check attachment_kind is unsupported")?; + } + if !null_or_one_of(value("run_scope_phase"), &CHECK_RUN_SCOPE_PHASES) { + reject("integration check run_scope_phase is unsupported")?; + } + for name in [ + "asset_name", + "attachment_kind", + "attached_column_name", + "attached_target_name", + "run_scope_phase", + "audit_definition_name", + ] { + optional_identifier(value(name), name)?; + } + let row_count: &JsonValue = value("row_count"); + if !matches!(row_count, JsonValue::Null) && !non_negative_integer(row_count) { + reject("integration check row_count must be non-negative")?; + } + if !matches!(value("reused"), JsonValue::Null | JsonValue::Bool(_)) { + reject("integration check reused must be boolean")?; + } + if !null_or_one_of(value("evaluation_mode"), &EVALUATION_MODES) { + reject("integration check evaluation_mode is unsupported")?; + } + let measured: &JsonValue = value("measured_value"); + if !matches!(measured, JsonValue::Null) && !finite_float(measured) { + reject("integration check measured_value must be finite")?; + } + for name in ["sample_count", "minimum_samples", "evidence_count"] { + let count: &JsonValue = value(name); + if !matches!(count, JsonValue::Null) && !non_negative_integer(count) { + reject(format!("integration check {name} must be non-negative"))?; + } + } + optional_identifier(value("sample_unit"), "sample_unit")?; + if !matches!( + value("evidence_truncated"), + JsonValue::Null | JsonValue::Bool(_) + ) { + reject("integration check evidence_truncated must be boolean")?; + } + let description: &JsonValue = value("audit_description"); + if !matches!(description, JsonValue::Null) + && !match description { + JsonValue::String(text) => !text.trim().is_empty(), + JsonValue::Utf16(_) => true, + _ => false, + } + { + reject("integration check audit_description must be a non-empty string")?; + } + if !matches!(value("thresholds"), JsonValue::Null) { + thresholds(value("thresholds"))?; + } + Ok(check) +} + +/// `integration_resource_id`: the canonical resource identifier of a result. +pub(crate) fn resource_id( + kind: &str, + name: &str, + check_id: Option<&str>, + loader: Option<&str>, +) -> String { + if CHECK_KINDS.contains(&kind) { + return check_id.unwrap_or_default().to_owned(); + } + if MODEL_RESOURCE_KINDS.contains(&kind) { + return format!("model:{name}"); + } + if kind == LOADER_KIND && loader.is_some() { + return format!("source:{name}"); + } + format!("{kind}:{name}") +} + +fn result_identity(record: &[(String, JsonValue)]) -> Result<(), IntegrationRecordError> { + let value = |key: &str| field(record, key); + let checks: &[JsonValue] = match value("checks") { + JsonValue::Array(checks) => checks, + _ => &[], + }; + let output_kind: Option<&str> = text(value("output_kind")); + if output_kind == Some(ASSET_OUTPUT) { + let JsonValue::Object(asset) = value("asset") else { + return reject("asset output must contain exactly one asset"); + }; + if !checks.is_empty() { + return reject("asset output must contain exactly one asset"); + } + if field(asset, "kind") != value("resource_kind") + || field(asset, "name") != value("resource_name") + { + return reject("integration asset identity does not match terminal"); + } + let expected: String = resource_id( + text(value("resource_kind")).unwrap_or_default(), + text(value("resource_name")).unwrap_or_default(), + None, + text(field(asset, "loader")), + ); + if text(value("resource_id")) != Some(expected.as_str()) { + return reject("integration asset resource_id is incoherent"); + } + return Ok(()); + } + if output_kind == Some(CHECK_OUTPUT) { + if !matches!(value("asset"), JsonValue::Null) || checks.len() != 1 { + return reject("check output must contain exactly one check"); + } + let JsonValue::Object(check) = &checks[0] else { + return reject("check output must contain exactly one check"); + }; + if field(check, "check_id") != value("resource_id") { + return reject("integration check identity does not match terminal"); + } + return Ok(()); + } + reject("integration result output_kind is unsupported") +} + +fn asset_status(record: &[(String, JsonValue)]) -> Option<&JsonValue> { + match field(record, "asset") { + JsonValue::Object(asset) => Some(field(asset, "status")), + _ => None, + } +} + +fn terminal_facts(record: &[(String, JsonValue)]) -> Result<(), IntegrationRecordError> { + let value = |key: &str| field(record, key); + let set = |key: &str| !matches!(value(key), JsonValue::Null); + let event_type: Option<&str> = text(value("event_type")); + if event_type == Some(RESOURCE_FAILED_EVENT) { + if !set("error_type") || set("skip_code") || set("skip_mode") { + return reject("failed integration terminal facts are invalid"); + } + if asset_status(record).is_some_and(|status| text(status) != Some(FAILED_STATUS)) { + return reject("failed terminal asset status is incoherent"); + } + return Ok(()); + } + if event_type == Some(RESOURCE_SKIPPED_EVENT) { + if !one_of(value("skip_code"), &RESOURCE_SKIP_CODES) + || set("error_code") + || set("error_type") + || !null_or_one_of(value("skip_mode"), &SKIP_MODES) + { + return reject("skipped integration terminal facts are invalid"); + } + if asset_status(record).is_some_and(|status| text(status) != Some(SKIPPED_STATUS)) { + return reject("skipped terminal asset status is incoherent"); + } + return Ok(()); + } + if ["error_code", "error_type", "skip_code", "skip_mode"] + .iter() + .any(|key| set(key)) + { + return reject("completed integration terminal has failure facts"); + } + Ok(()) +} + +/// The `IntegrationResultEnvelope` checks of a completed record. +pub(crate) fn envelope(record: &[(String, JsonValue)]) -> Result<(), IntegrationRecordError> { + let value = |key: &str| field(record, key); + if !one_of(value("event_type"), &RESOURCE_TERMINALS) { + return reject("integration result event_type is not a resource terminal"); + } + let identity: [&str; 10] = [ + "event_id", + "event_type", + "occurred_at", + "invocation_id", + "run_id", + "resource_id", + "resource_attempt_id", + "resource_kind", + "resource_name", + "command", + ]; + if identity + .iter() + .any(|key| text_length(value(key)).is_none_or(|length| length == 0)) + { + return reject("integration result identity fields must be non-empty"); + } + for key in identity { + if !NON_IDENTIFIER_IDENTITY_KEYS.contains(&key) { + require_identifier(value(key), key)?; + } + } + if !one_of( + value("resource_kind"), + &crate::integration_results::constants::ASSET_KINDS, + ) && !one_of(value("resource_kind"), &EXTRA_RESOURCE_KINDS) + { + return reject("integration resource_kind is unsupported"); + } + optional_identifier(value("operation_id"), "operation_id")?; + optional_identifier(value("statement_id"), "statement_id")?; + match text(value("occurred_at")).and_then(aware_timestamp) { + Some(true) => {} + Some(false) => return reject("integration result occurred_at must include a timezone"), + None => return reject("integration result occurred_at must be an ISO-8601 timestamp"), + } + if !non_negative_integer(value("event_sequence")) { + return reject("integration result event_sequence must be non-negative"); + } + let attempt_positive: bool = match value("attempt_number") { + JsonValue::Integer(number) => number.to_i128().is_none_or(|number| number >= 1), + _ => false, + }; + if !attempt_positive { + return reject("integration result attempt_number must be positive"); + } + let no_checks: bool = matches!(value("checks"), JsonValue::Array(checks) if checks.is_empty()); + if matches!(value("asset"), JsonValue::Null) && no_checks { + return reject("integration result must contain asset or check enrichment"); + } + let duration_valid: bool = match value("duration_ms") { + JsonValue::Null => true, + JsonValue::Integer(number) => !number.as_str().starts_with('-'), + JsonValue::Float(number) => number.is_finite() && *number >= MIN_DURATION_MS, + _ => false, + }; + if !duration_valid { + return reject("integration result duration_ms must be non-negative"); + } + for key in ["error_code", "error_type", "skip_code", "skip_mode"] { + optional_identifier(value(key), key)?; + } + if !matches!(value("projection_degraded"), JsonValue::Bool(_)) { + return reject("integration result projection_degraded must be boolean"); + } + result_identity(record)?; + terminal_facts(record) +} + +/// The envelope fields followed by `projection_degraded`, the asset and the checks. +pub(crate) fn envelope_fields() -> Vec<(&'static str, bool)> { + let mut fields: Vec<(&'static str, bool)> = ENVELOPE_FIELDS.to_vec(); + fields.push(("projection_degraded", false)); + fields +} diff --git a/crates/sqlbuild-core/src/integration_results/_helpers/timestamps.rs b/crates/sqlbuild-core/src/integration_results/_helpers/timestamps.rs new file mode 100644 index 0000000000..93b465432a --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/_helpers/timestamps.rs @@ -0,0 +1,69 @@ +//! `datetime.fromisoformat` for the timestamps integration records carry. + +use crate::integration_results::constants::{MAX_DAY, MAX_HOUR, MAX_MINUTE_OR_SECOND, MAX_MONTH}; + +fn digits(text: &[u8], count: usize) -> Option<(u32, &[u8])> { + if text.len() < count || !text[..count].iter().all(u8::is_ascii_digit) { + return None; + } + let value: u32 = text[..count] + .iter() + .fold(0, |total, digit| total * 10 + u32::from(digit - b'0')); + Some((value, &text[count..])) +} + +/// `HH[:MM[:SS[.ffffff]]]` of a time or offset, returning the rest. +fn clock(text: &[u8]) -> Option<&[u8]> { + let (hour, mut rest) = digits(text, 2)?; + if hour > MAX_HOUR { + return None; + } + for limit in [MAX_MINUTE_OR_SECOND, MAX_MINUTE_OR_SECOND] { + let Some(after) = rest.strip_prefix(b":") else { + break; + }; + let (value, after) = digits(after, 2)?; + if value > limit { + return None; + } + rest = after; + } + if let Some(after) = rest.strip_prefix(b".").or_else(|| rest.strip_prefix(b",")) { + let length: usize = after + .iter() + .take_while(|byte| byte.is_ascii_digit()) + .count(); + if length == 0 { + return None; + } + rest = &after[length..]; + } + Some(rest) +} + +/// Whether the text is an ISO 8601 date-time `fromisoformat` reads with a UTC offset. +pub(crate) fn aware_timestamp(text: &str) -> Option { + let bytes: &[u8] = text.as_bytes(); + let (_, rest) = digits(bytes, 4)?; + let rest: &[u8] = rest.strip_prefix(b"-")?; + let (month, rest) = digits(rest, 2)?; + let rest: &[u8] = rest.strip_prefix(b"-")?; + let (day, rest) = digits(rest, 2)?; + if !(1..=MAX_MONTH).contains(&month) || !(1..=MAX_DAY).contains(&day) { + return None; + } + if rest.is_empty() { + return Some(false); + } + let rest: &[u8] = clock(&rest[1..])?; + if rest.is_empty() { + return Some(false); + } + if rest == b"Z" { + return Some(true); + } + let offset: &[u8] = rest + .strip_prefix(b"+") + .or_else(|| rest.strip_prefix(b"-"))?; + clock(offset)?.is_empty().then_some(true) +} diff --git a/crates/sqlbuild-core/src/integration_results/_helpers/values.rs b/crates/sqlbuild-core/src/integration_results/_helpers/values.rs new file mode 100644 index 0000000000..0a94288347 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/_helpers/values.rs @@ -0,0 +1,77 @@ +//! Reading JSON values with Python's type distinctions. + +use crate::integration_results::constants::MAX_IDENTIFIER_CHARS; +use crate::integration_results::errors::IntegrationRecordError; +use crate::json::models::JsonValue; + +/// Python `len()` of a string value, or `None` when the value is not a string. +pub(crate) fn text_length(value: &JsonValue) -> Option { + match value { + JsonValue::String(text) => Some(text.chars().count()), + JsonValue::Utf16(units) => Some(char::decode_utf16(units.iter().copied()).count()), + _ => None, + } +} + +/// The text of a string value without lone surrogates. +pub(crate) fn text(value: &JsonValue) -> Option<&str> { + match value { + JsonValue::String(text) => Some(text), + _ => None, + } +} + +/// Whether the value is a string equal to one of `allowed`. +pub(crate) fn one_of(value: &JsonValue, allowed: &[&str]) -> bool { + text(value).is_some_and(|text| allowed.contains(&text)) +} + +/// The value of `key` in an object, or `Null` when absent. +pub(crate) fn field<'a>(object: &'a [(String, JsonValue)], key: &str) -> &'a JsonValue { + object + .iter() + .find(|(name, _)| name == key) + .map_or(&JsonValue::Null, |(_, value)| value) +} + +/// A non-negative Python `int` (never `bool`). +pub(crate) fn non_negative_integer(value: &JsonValue) -> bool { + match value { + JsonValue::Integer(number) => !number.as_str().starts_with('-'), + _ => false, + } +} + +/// A finite Python `float` (never `int`). +pub(crate) fn finite_float(value: &JsonValue) -> bool { + matches!(value, JsonValue::Float(number) if number.is_finite()) +} + +/// `validate_identifier`: a non-empty string of at most the identifier limit. +pub(crate) fn require_identifier( + value: &JsonValue, + field_name: &str, +) -> Result<(), IntegrationRecordError> { + match text_length(value) { + Some(length) if length > 0 && length <= MAX_IDENTIFIER_CHARS => Ok(()), + _ => Err(IntegrationRecordError::new(format!( + "integration {field_name} is invalid" + ))), + } +} + +/// `validate_optional_identifier`: `null`, or an identifier. +pub(crate) fn optional_identifier( + value: &JsonValue, + field_name: &str, +) -> Result<(), IntegrationRecordError> { + if matches!(value, JsonValue::Null) { + return Ok(()); + } + require_identifier(value, field_name) +} + +/// The value is `null` or one of `allowed`. +pub(crate) fn null_or_one_of(value: &JsonValue, allowed: &[&str]) -> bool { + matches!(value, JsonValue::Null) || one_of(value, allowed) +} diff --git a/crates/sqlbuild-core/src/integration_results/constants.rs b/crates/sqlbuild-core/src/integration_results/constants.rs new file mode 100644 index 0000000000..512d74e0aa --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/constants.rs @@ -0,0 +1,252 @@ +//! The integration-result record contract: identity, limits and allowed values. + +pub(crate) const SCHEMA_VERSION: i64 = 1; +pub(crate) const RECORD_KIND: &str = "integration_result"; +pub(crate) const RESOURCE_FAILED_EVENT: &str = "resource_attempt_failed"; +pub(crate) const RESOURCE_SKIPPED_EVENT: &str = "resource_attempt_skipped"; +pub(crate) const RESOURCE_TERMINALS: [&str; 3] = [ + "resource_attempt_completed", + RESOURCE_FAILED_EVENT, + RESOURCE_SKIPPED_EVENT, +]; +pub(crate) const RESOURCE_SKIP_CODES: [&str; 4] = ["dependency", "explicit", "fan_in", "scheduler"]; +pub(crate) const FAILED_STATUS: &str = "failed"; +pub(crate) const SKIPPED_STATUS: &str = "skipped"; +pub(crate) const LOADER_KIND: &str = "loader"; +pub(crate) const AUDIT_KIND: &str = "audit"; +pub(crate) const ASSET_OUTPUT: &str = "asset"; +pub(crate) const CHECK_OUTPUT: &str = "check"; +pub(crate) const MAX_RECORD_BYTES: usize = 65_536; +pub(crate) const MAX_METADATA_BYTES: usize = 16_384; +pub(crate) const MAX_STRING_CHARS: usize = 2_048; +pub(crate) const MAX_IDENTIFIER_CHARS: usize = 512; +pub(crate) const MAX_COLLECTION_ITEMS: usize = 128; +pub(crate) const MAX_NESTING_DEPTH: usize = 6; +pub(crate) const ASSET_KINDS: [&str; 12] = [ + "asset", "custom", "loader", "model", "seed", "snapshot", "source", "table", "table_fn", + "task", "udf", "view", +]; +pub(crate) const CHECK_KINDS: [&str; 3] = [AUDIT_KIND, "python_check", "sql_test"]; +pub(crate) const EXTRA_RESOURCE_KINDS: [&str; 3] = ["audit", "check", "test"]; +pub(crate) const MODEL_RESOURCE_KINDS: [&str; 5] = ["custom", "model", "snapshot", "table", "view"]; +pub(crate) const ASSET_STATUSES: [&str; 4] = ["failed", "skipped", "success", "warning"]; +pub(crate) const FAILED_PHASES: [&str; 12] = [ + "audit", + "contract", + "custom_materialization", + "dml", + "fingerprint", + "microbatch_state", + "post_hook", + "pre_hook", + "promotion", + "schema_change", + "staging", + "type_enforcement", +]; +pub(crate) const CHECK_STATUSES: [&str; 5] = ["error", "fail", "insufficient", "pass", "warn"]; +pub(crate) const CHECK_NON_FAIL_STATUSES: [&str; 2] = ["insufficient", "pass"]; +/// `AuditSeverity` values, in declaration order. +pub(crate) const AUDIT_SEVERITIES: [&str; 2] = ["warn", "error"]; +pub(crate) const CHECK_ATTACHMENT_KINDS: [&str; 3] = ["end", "model", "source"]; +pub(crate) const CHECK_RUN_SCOPE_PHASES: [&str; 2] = ["delta_and_final", "final"]; +pub(crate) const EVALUATION_MODES: [&str; 2] = ["violations", "measurement"]; +pub(crate) const BOUND_OPERATORS: [&str; 2] = ["below", "above"]; +pub(crate) const RANGE_OPERATOR: &str = "outside"; +pub(crate) const CLONE_ACTIONS: [&str; 7] = [ + "cloned", + "copied", + "recreated_view", + "recreated_function", + "warning_missing_source", + "skipped_missing_dependency", + "failed", +]; +pub(crate) const SKIP_MODES: [&str; 2] = ["hard", "soft"]; +pub(crate) const FUTURE_CURSOR_KEYS: [&str; 16] = [ + "action", + "applied_bounds", + "cursor_column", + "determining_input", + "discovered_bounds", + "end", + "future_end_detected", + "future_start_detected", + "inputs", + "invocation_time", + "max_distance", + "maximum", + "maximum_allowed_bounds", + "minimum", + "relation", + "start", +]; +pub(crate) const MAXIMUM_START_KEYS: [&str; 10] = [ + "action", + "cursor_column", + "effective_start", + "highest_eligible_target_max", + "input", + "invocation_time", + "max_ahead", + "maximum_allowed_start", + "physical_target_max", + "relation", +]; +pub(crate) const MAXIMUM_START_ACTIONS: [&str; 2] = ["cap", "error"]; +pub(crate) const MAXIMUM_START_INPUT_KEYS: [&str; 2] = ["relation", "cursor_column"]; +pub(crate) const MAXIMUM_START_REQUIRED_KEYS: [&str; 8] = [ + "action", + "max_ahead", + "invocation_time", + "physical_target_max", + "highest_eligible_target_max", + "effective_start", + "maximum_allowed_start", + "input", +]; +pub(crate) const MICROBATCH_KEYS: [&str; 22] = [ + "action", + "batch_concurrency", + "batch_count", + "batch_size", + "concurrent_enabled", + "contiguous_frontier", + "count", + "global_concurrency", + "known_gap_count", + "limit", + "physical_generation_id", + "recovery_batch_count", + "replay_requirement_id", + "replay_requirement_state", + "required_model_version_hash", + "run_type", + "strategy", + "reason", + "synthetic_completion_count", + "unaccounted_interval_count", + "unaccounted_partition_policy", + "unknown_fingerprint_count", +]; +pub(crate) const MICROBATCH_COUNT_KEYS: [&str; 10] = [ + "batch_concurrency", + "batch_count", + "count", + "global_concurrency", + "known_gap_count", + "limit", + "recovery_batch_count", + "synthetic_completion_count", + "unaccounted_interval_count", + "unknown_fingerprint_count", +]; +pub(crate) const MICROBATCH_RUN_TYPES: [&str; 3] = ["backfill", "normal", "replay_on_change"]; +pub(crate) const MICROBATCH_LIMIT_ACTIONS: [&str; 4] = + ["cap_from_end", "cap_from_start", "error", "warn"]; +pub(crate) const MICROBATCH_PARTITION_POLICIES: [&str; 3] = + ["recover_all", "recover_empty", "synthesize"]; +pub(crate) const MICROBATCH_REPLAY_STATES: [&str; 4] = [ + "complete_with_unknown_fingerprints", + "incomplete", + "superseded", + "verified_complete", +]; +/// Asset fields in record order, with whether each is required. +pub(crate) const ASSET_FIELDS: [(&str, bool); 14] = [ + ("kind", true), + ("name", true), + ("status", true), + ("action", false), + ("target", false), + ("origin_relation", false), + ("staging_relation", false), + ("failed_phase", false), + ("loader", false), + ("rows_loaded", false), + ("materialized", false), + ("future_cursor_safety", false), + ("maximum_start_safety", false), + ("microbatch", false), +]; +/// Asset fields whose default is an empty object rather than `null`. +pub(crate) const ASSET_MAPPING_FIELDS: [&str; 3] = + ["future_cursor_safety", "maximum_start_safety", "microbatch"]; +pub(crate) const CHECK_FIELDS: [(&str, bool); 24] = [ + ("kind", true), + ("name", true), + ("check_id", true), + ("passed", true), + ("status", true), + ("dag_check_id", false), + ("severity", false), + ("asset_name", false), + ("attachment_kind", false), + ("attached_column_name", false), + ("attached_target_name", false), + ("run_scope_phase", false), + ("row_count", false), + ("reused", false), + ("evaluation_mode", false), + ("measured_value", false), + ("sample_count", false), + ("sample_unit", false), + ("minimum_samples", false), + ("thresholds", false), + ("evidence_count", false), + ("evidence_truncated", false), + ("audit_definition_name", false), + ("audit_description", false), +]; +/// Envelope fields in record order, before `projection_degraded`, `asset` and `checks`. +pub(crate) const ENVELOPE_FIELDS: [(&str, bool); 22] = [ + ("schema_version", true), + ("record_kind", true), + ("event_id", true), + ("event_sequence", true), + ("event_type", true), + ("occurred_at", true), + ("invocation_id", true), + ("run_id", true), + ("resource_id", true), + ("resource_attempt_id", true), + ("operation_id", true), + ("statement_id", true), + ("resource_kind", true), + ("resource_name", true), + ("attempt_number", true), + ("duration_ms", true), + ("output_kind", true), + ("command", true), + ("error_code", false), + ("error_type", false), + ("skip_code", false), + ("skip_mode", false), +]; +/// Asset enrichment a reader may drop to accept a newer writer's record. +pub(crate) const OPTIONAL_ASSET_ENRICHMENT: [&str; 4] = [ + "action", + "failed_phase", + "future_cursor_safety", + "maximum_start_safety", +]; +pub(crate) const OPTIONAL_MICROBATCH_ENUMS: [&str; 4] = [ + "action", + "replay_requirement_state", + "run_type", + "unaccounted_partition_policy", +]; +pub(crate) const OPTIONAL_CHECK_ENRICHMENT: [&str; 3] = + ["attachment_kind", "run_scope_phase", "severity"]; +pub(crate) const PROJECTION_DEGRADED_FIELD: &str = "projection_degraded"; +pub(crate) const THRESHOLD_LEVELS: [&str; 2] = ["warn", "error"]; +/// Maximum-start fields whose values are not plain timestamps. +pub(crate) const MAXIMUM_START_STRUCTURED_KEYS: [&str; 2] = ["action", "input"]; +pub(crate) const MAXIMUM_START_NULLABLE_KEY: &str = "highest_eligible_target_max"; +/// Identity fields that are not identifiers: the terminal type and its timestamp. +pub(crate) const NON_IDENTIFIER_IDENTITY_KEYS: [&str; 2] = ["event_type", "occurred_at"]; +pub(crate) const MAX_HOUR: u32 = 23; +pub(crate) const MAX_MINUTE_OR_SECOND: u32 = 59; +pub(crate) const MAX_MONTH: u32 = 12; +pub(crate) const MAX_DAY: u32 = 31; +pub(crate) const MIN_DURATION_MS: f64 = 0.0; diff --git a/crates/sqlbuild-core/src/integration_results/errors.rs b/crates/sqlbuild-core/src/integration_results/errors.rs new file mode 100644 index 0000000000..08518e4bf5 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/errors.rs @@ -0,0 +1,23 @@ +//! A record that violates the integration-result contract. + +use std::fmt; + +/// The contract an integration-result record or value broke, as Python's `ObservabilityValidationError` said it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IntegrationRecordError { + pub message: String, +} + +impl IntegrationRecordError { + pub(crate) fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl fmt::Display for IntegrationRecordError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.message) + } +} diff --git a/crates/sqlbuild-core/src/integration_results/main/encode_integration_value.rs b/crates/sqlbuild-core/src/integration_results/main/encode_integration_value.rs new file mode 100644 index 0000000000..bf9106ac60 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/main/encode_integration_value.rs @@ -0,0 +1,13 @@ +//! Strict bounded encoding of integration values and records. + +use crate::integration_results::_helpers::bounded::encode; +use crate::integration_results::errors::IntegrationRecordError; +use crate::json::models::JsonValue; + +/// Compact ASCII JSON of a bounded value, within the record limit when `record` is set, else the metadata limit. +pub fn encode_integration_value( + value: &JsonValue, + record: bool, +) -> Result { + encode(value, record) +} diff --git a/crates/sqlbuild-core/src/integration_results/main/integration_record_fits.rs b/crates/sqlbuild-core/src/integration_results/main/integration_record_fits.rs new file mode 100644 index 0000000000..fdc54c174c --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/main/integration_record_fits.rs @@ -0,0 +1,9 @@ +//! The size limit a reader applies before decoding a record. + +use crate::integration_results::constants::MAX_RECORD_BYTES; + +/// Whether a record of `byte_length` UTF-8 bytes is within the record limit. +#[must_use] +pub fn integration_record_fits(byte_length: usize) -> bool { + byte_length <= MAX_RECORD_BYTES +} diff --git a/crates/sqlbuild-core/src/integration_results/main/mod.rs b/crates/sqlbuild-core/src/integration_results/main/mod.rs new file mode 100644 index 0000000000..abe95225db --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/main/mod.rs @@ -0,0 +1,4 @@ +pub mod encode_integration_value; +pub mod integration_record_fits; +pub mod normalize_integration_record; +pub mod strip_optional_enrichment; diff --git a/crates/sqlbuild-core/src/integration_results/main/normalize_integration_record.rs b/crates/sqlbuild-core/src/integration_results/main/normalize_integration_record.rs new file mode 100644 index 0000000000..5365dbf726 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/main/normalize_integration_record.rs @@ -0,0 +1,29 @@ +//! Decode one integration-result record into its complete canonical form. + +use crate::integration_results::_helpers::decoding::{ + RecordParts, decoded_asset, decoded_checks, require_supported, split, with_degraded_default, +}; +use crate::integration_results::_helpers::records::{completed, envelope, envelope_fields}; +use crate::integration_results::errors::IntegrationRecordError; +use crate::json::models::JsonValue; + +/// `IntegrationResultEnvelope.from_json` after JSON parsing: the record with every field, in order. +pub fn normalize_integration_record( + record: JsonValue, +) -> Result, IntegrationRecordError> { + let JsonValue::Object(entries) = record else { + return Err(IntegrationRecordError::new( + "integration result must be a JSON object", + )); + }; + require_supported(&entries)?; + let parts: RecordParts = split(entries)?; + let fields: Vec<(String, JsonValue)> = with_degraded_default(parts.envelope)?; + let asset: JsonValue = decoded_asset(parts.asset)?; + let checks: Vec = decoded_checks(parts.checks)?; + let mut record: Vec<(String, JsonValue)> = completed(fields, &envelope_fields(), &[])?; + record.push(("asset".to_owned(), asset)); + record.push(("checks".to_owned(), JsonValue::Array(checks))); + envelope(&record)?; + Ok(record) +} diff --git a/crates/sqlbuild-core/src/integration_results/main/strip_optional_enrichment.rs b/crates/sqlbuild-core/src/integration_results/main/strip_optional_enrichment.rs new file mode 100644 index 0000000000..def230a222 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/main/strip_optional_enrichment.rs @@ -0,0 +1,13 @@ +//! Drop forward-compatible optional enrichment so an older reader can accept a record. + +use crate::integration_results::_helpers::enrichment::stripped_field; +use crate::json::models::JsonValue; + +/// The record without optional asset, microbatch-enum and check enrichment. +#[must_use] +pub fn strip_optional_enrichment(record: JsonValue) -> JsonValue { + let JsonValue::Object(entries) = record else { + return record; + }; + JsonValue::Object(entries.into_iter().map(stripped_field).collect()) +} diff --git a/crates/sqlbuild-core/src/integration_results/mod.rs b/crates/sqlbuild-core/src/integration_results/mod.rs new file mode 100644 index 0000000000..18dc6143a5 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/integration_results/tests.rs b/crates/sqlbuild-core/src/integration_results/tests.rs new file mode 100644 index 0000000000..6f638c9065 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_integration_results.rs"] +mod integration_results; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/integration_results/tests/helpers.rs b/crates/sqlbuild-core/src/integration_results/tests/helpers.rs new file mode 100644 index 0000000000..53f50afe2f --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/tests/helpers.rs @@ -0,0 +1,52 @@ +use crate::json::models::{JsonInteger, JsonValue}; + +pub(super) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +pub(super) fn integer(value: i64) -> JsonValue { + JsonValue::Integer(JsonInteger::from(value)) +} + +pub(super) fn object(entries: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + entries + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + ) +} + +/// A completed model asset record with the given asset and occurrence time. +pub(super) fn record(asset: JsonValue, occurred_at: &str) -> JsonValue { + object(vec![ + ("schema_version", integer(1)), + ("record_kind", text("integration_result")), + ("event_id", text("e1")), + ("event_sequence", integer(0)), + ("event_type", text("resource_attempt_completed")), + ("occurred_at", text(occurred_at)), + ("invocation_id", text("i1")), + ("run_id", text("r1")), + ("resource_id", text("model:orders")), + ("resource_attempt_id", text("a1")), + ("operation_id", JsonValue::Null), + ("statement_id", JsonValue::Null), + ("resource_kind", text("model")), + ("resource_name", text("orders")), + ("attempt_number", integer(1)), + ("duration_ms", integer(3)), + ("output_kind", text("asset")), + ("command", text("build")), + ("asset", asset), + ]) +} + +pub(super) fn orders_asset(status: &str) -> JsonValue { + object(vec![ + ("kind", text("model")), + ("name", text("orders")), + ("status", text(status)), + ("microbatch", object(vec![("run_type", text("normal"))])), + ]) +} diff --git a/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs b/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs new file mode 100644 index 0000000000..5dbf05e0a4 --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs @@ -0,0 +1,101 @@ +use crate::integration_results::main::encode_integration_value::encode_integration_value; +use crate::integration_results::main::normalize_integration_record::normalize_integration_record; +use crate::integration_results::main::strip_optional_enrichment::strip_optional_enrichment; +use crate::integration_results::tests::helpers::{integer, object, orders_asset, record, text}; +use crate::integration_results::tests::test_types::NormalizeTestCase; +use crate::json::models::JsonValue; + +#[test] +fn given_integration_records_when_normalizing_then_contract_is_enforced() { + let test_cases = [ + NormalizeTestCase { + description: "a minimal asset record gains every default in field order", + record: record(orders_asset("success"), "2026-09-02T12:00:00.123456+00:00"), + expected_error: None, + expected_encoded: Some( + "{\"schema_version\":1,\"record_kind\":\"integration_result\",\"event_id\":\"e1\",\"event_sequence\":0,\"event_type\":\"resource_attempt_completed\",\"occurred_at\":\"2026-09-02T12:00:00.123456+00:00\",\"invocation_id\":\"i1\",\"run_id\":\"r1\",\"resource_id\":\"model:orders\",\"resource_attempt_id\":\"a1\",\"operation_id\":null,\"statement_id\":null,\"resource_kind\":\"model\",\"resource_name\":\"orders\",\"attempt_number\":1,\"duration_ms\":3,\"output_kind\":\"asset\",\"command\":\"build\",\"error_code\":null,\"error_type\":null,\"skip_code\":null,\"skip_mode\":null,\"projection_degraded\":false,\"asset\":{\"kind\":\"model\",\"name\":\"orders\",\"status\":\"success\",\"action\":null,\"target\":null,\"origin_relation\":null,\"staging_relation\":null,\"failed_phase\":null,\"loader\":null,\"rows_loaded\":null,\"materialized\":null,\"future_cursor_safety\":{},\"maximum_start_safety\":{},\"microbatch\":{\"run_type\":\"normal\"}},\"checks\":[]}", + ), + }, + NormalizeTestCase { + description: "a naive occurrence time is rejected", + record: record(orders_asset("success"), "2026-09-02T12:00:00"), + expected_error: Some("integration result occurred_at must include a timezone"), + expected_encoded: None, + }, + NormalizeTestCase { + description: "an unknown asset status is rejected", + record: record(orders_asset("pending"), "2026-09-02T12:00:00Z"), + expected_error: Some("integration asset status is unsupported"), + expected_encoded: None, + }, + NormalizeTestCase { + description: "an unknown asset field makes the envelope invalid", + record: record( + object(vec![ + ("kind", text("model")), + ("name", text("orders")), + ("status", text("success")), + ("colour", text("blue")), + ]), + "2026-09-02T12:00:00Z", + ), + expected_error: Some("invalid integration result envelope"), + expected_encoded: None, + }, + ]; + + for test_case in test_cases { + let normalized = normalize_integration_record(test_case.record); + let error: Option = normalized.as_ref().err().map(|error| error.message.clone()); + let encoded: Option = normalized + .ok() + .and_then(|record| encode_integration_value(&JsonValue::Object(record), true).ok()); + assert_eq!( + (error.as_deref(), encoded.as_deref()), + (test_case.expected_error, test_case.expected_encoded), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_newer_enrichment_when_stripping_then_record_keeps_core_fields() { + let test_cases = [NormalizeTestCase { + description: "optional asset and microbatch enum enrichment is dropped", + record: record( + object(vec![ + ("kind", text("model")), + ("name", text("orders")), + ("status", text("success")), + ("failed_phase", text("future_phase")), + ( + "microbatch", + object(vec![("run_type", text("new")), ("batch_count", integer(2))]), + ), + ]), + "2026-09-02T12:00:00Z", + ), + expected_error: None, + expected_encoded: Some("{\"batch_count\":2}"), + }]; + + for test_case in test_cases { + let normalized = normalize_integration_record(strip_optional_enrichment(test_case.record)); + let error: Option = normalized.as_ref().err().map(|error| error.message.clone()); + let microbatch: Option = normalized.ok().and_then(|record| { + let asset: JsonValue = record.into_iter().find(|(key, _)| key == "asset")?.1; + let JsonValue::Object(asset) = asset else { + return None; + }; + let microbatch: JsonValue = asset.into_iter().find(|(key, _)| key == "microbatch")?.1; + encode_integration_value(µbatch, false).ok() + }); + assert_eq!( + (error.as_deref(), microbatch.as_deref()), + (test_case.expected_error, test_case.expected_encoded), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/integration_results/tests/test_types.rs b/crates/sqlbuild-core/src/integration_results/tests/test_types.rs new file mode 100644 index 0000000000..bacd65819f --- /dev/null +++ b/crates/sqlbuild-core/src/integration_results/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::json::models::JsonValue; + +pub(super) struct NormalizeTestCase { + pub(super) description: &'static str, + pub(super) record: JsonValue, + pub(super) expected_error: Option<&'static str>, + pub(super) expected_encoded: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/counts.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/counts.rs new file mode 100644 index 0000000000..5dff531a2c --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/counts.rs @@ -0,0 +1,35 @@ +//! Counts and labels the janitor confirmation and summary share. + +use crate::janitor_text::constants::DEFAULT_ENVIRONMENT; +use crate::janitor_text::models::JanitorPlanText; + +/// The plan's target name, or `default`. +pub(crate) fn environment_label(plan: &JanitorPlanText) -> &str { + plan.target_name.as_deref().unwrap_or(DEFAULT_ENVIRONMENT) +} + +/// Warehouse relations the plan drops. +pub(crate) fn physical_deletion_count(plan: &JanitorPlanText) -> usize { + let relations: usize = if plan.direct_mode { + plan.archive_deletions.len() + } else { + plan.candidates.len() + }; + relations + plan.query_artifacts.len() + plan.old_name_drops.len() +} + +/// `archive N and ` when the plan archives relations. +pub(crate) fn archive_prefix(plan: &JanitorPlanText) -> String { + if plan.archive_candidates.is_empty() { + return String::new(); + } + format!("archive {} and ", plan.archive_candidates.len()) +} + +/// The connection an `--as` preview inspects through. +pub(crate) fn connection_label(active_target: Option<&str>) -> String { + active_target.map_or_else( + || "the active connection".to_owned(), + |target| format!("the connection of target '{target}'"), + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs new file mode 100644 index 0000000000..c6213b2bdb --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod counts; +pub(crate) mod plan_sections; diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs new file mode 100644 index 0000000000..87d0aeb42f --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs @@ -0,0 +1,286 @@ +//! Sections of the janitor plan preview. + +use crate::cli_text::models::CliStyle; +use crate::janitor_text::constants::{ + DROP_EARLY, DROP_EXPIRED, DROP_MISSING, LABEL_WIDTH, TREE_BLANK, TREE_BRANCH_GLYPH, + TREE_CONTINUATION, TREE_LAST_GLYPH, +}; +use crate::janitor_text::models::{JanitorPlanText, OldNameViewText}; + +fn row(label: &str, rendered: &str) -> String { + format!(" {label:(style: CliStyle, label: &str, items: &[T]) -> String { + let count: String = items.len().to_string(); + let rendered: String = if items.is_empty() { + style.accent(&count) + } else { + style.warning(&count) + }; + row(label, &rendered) +} + +fn tree_connector(style: CliStyle, last: bool) -> String { + style.muted(if last { + TREE_LAST_GLYPH + } else { + TREE_BRANCH_GLYPH + }) +} + +/// `age Nd`, or `age unknown`. +fn age(days: Option) -> String { + days.map_or_else( + || "age unknown".to_owned(), + |days| format!("age {}d", days.max(0)), + ) +} + +/// The retention, schema and per-action counts. +pub(crate) fn summary(plan: &JanitorPlanText, style: CliStyle) -> String { + let mut text: String = String::new(); + if plan.retention_days == 0 { + text.push_str(&row("retention", &style.accent("disabled (0 days)"))); + text.push_str(&row("age metadata", &style.accent("not checked"))); + } else { + text.push_str(&row( + "retention", + &style.accent(&format!("{} days", plan.retention_days)), + )); + if !plan.age_metadata_supported { + text.push_str(&row("age metadata", &style.accent("unavailable"))); + } + } + if plan.direct_mode { + text.push_str(&row( + "archive retention", + &style.accent(&format!("{} days", plan.archive_retention_days)), + )); + } + text.push_str(&row( + "schemas scanned", + &style.accent(&plan.scanned_schema_count.to_string()), + )); + text.push_str(&row( + "schemas skipped", + &style.accent(&plan.skipped_schemas.len().to_string()), + )); + if plan.direct_mode { + text.push_str(&count_row( + style, + "relations to archive", + &plan.archive_candidates, + )); + text.push_str(&count_row( + style, + "archives to delete", + &plan.archive_deletions, + )); + text.push_str(&row( + "archives retained", + &style.accent(&plan.retained_archives.len().to_string()), + )); + text.push_str(&row( + "old name views", + &style.accent(&plan.old_name_live.len().to_string()), + )); + text.push_str(&count_row( + style, + "old name views to drop", + &plan.old_name_drops, + )); + } else { + text.push_str(&count_row(style, "eligible for deletion", &plan.candidates)); + } + text.push_str(&count_row(style, "query artifacts", &plan.query_artifacts)); + text.push_str(&count_row(style, "direct state pruned", &plan.state_prunes)); + text.push_str(&row( + "objects skipped", + &style.accent(&plan.skipped_relations.len().to_string()), + )); + text +} + +/// Schemas whose active sources block every janitor action. +pub(crate) fn blocked(plan: &JanitorPlanText, style: CliStyle) -> String { + if plan.blocked_schemas.is_empty() { + return String::new(); + } + let action: &str = if plan.direct_mode { + "archive" + } else { + "deletion" + }; + let mut text: String = format!( + "\n{}\n {}\n", + style.error_strong("Janitor blocked"), + style.error("Managed target schemas contain active configured sources.") + ); + for schema in &plan.blocked_schemas { + text.push_str(&format!( + " {} {}\n", + style.object_name(&schema.name), + style.error_muted(&format!( + "active sources: {}", + schema.source_names.join(", ") + )) + )); + for name in &schema.suppressed_candidates { + text.push_str(&format!( + " {}\n", + style.error_muted(&format!("suppressed {action}: {name}")) + )); + } + for name in &schema.suppressed_archive_deletions { + text.push_str(&format!( + " {}\n", + style.error_muted(&format!("suppressed archive deletion: {name}")) + )); + } + } + text.push_str(&format!( + " {}\n", + style.error("No janitor actions will be performed.") + )); + text +} + +/// Relations to archive, archives to delete, and retained archives. +pub(crate) fn archives(plan: &JanitorPlanText, style: CliStyle) -> String { + let mut text: String = String::new(); + if !plan.archive_candidates.is_empty() { + text.push_str(&format!("\n{}\n", style.success("Relations to archive"))); + for candidate in &plan.archive_candidates { + let expiry: String = if candidate.expires_now { + "deleted in this run".to_owned() + } else { + format!("delete after {} UTC", candidate.expires_at) + }; + text.push_str(&format!( + " {} -> {} {}\n", + style.object_name(&candidate.name), + style.object_name(&candidate.archive_name), + style.muted(&format!("{}, {expiry}", age(candidate.age_days))) + )); + } + } + if !plan.archive_deletions.is_empty() { + text.push_str(&format!("\n{}\n", style.success("Archives to delete"))); + for deletion in &plan.archive_deletions { + let detail: String = if deletion.archived_in_run { + "archived in this run".to_owned() + } else { + format!( + "archived {} UTC, {}, expired {} UTC", + deletion.archived_at, + age(deletion.age_days), + deletion.expires_at + ) + }; + text.push_str(&format!( + " {} {}\n", + style.object_name(&deletion.name), + style.muted(&detail) + )); + } + } + if !plan.retained_archives.is_empty() { + text.push_str(&format!("\n{}\n", style.success("Retained archives"))); + for retained in &plan.retained_archives { + text.push_str(&format!( + " {} {}\n", + style.object_name(&retained.name), + style.muted(&format!( + "archived {} UTC, expires {} UTC", + retained.archived_at, retained.expires_at + )) + )); + } + } + text +} + +fn old_name_detail(view: &OldNameViewText) -> (&'static str, String) { + let expiry: &str = view.expires_on.as_deref().unwrap_or_default(); + match view.drop_reason.as_deref() { + Some(DROP_EARLY) => ("drop", format!("now (requested; would expire {expiry})")), + Some(DROP_EXPIRED) => ("drop", format!("now (expired {expiry})")), + _ if view.occupied => { + let owner: String = view + .claimed_by + .as_ref() + .map_or_else(String::new, |model| format!(", model:{model}")); + ( + "record", + format!("dropped (name now used by another relation{owner})"), + ) + } + Some(DROP_MISSING) => ("record", "dropped (the view no longer exists)".to_owned()), + _ => ( + "expires", + format!( + "{expiry} (live; drop early with --drop-old-name-view {})", + view.name + ), + ), + } +} + +/// Compatibility views at old model names and what happens to each. +pub(crate) fn old_name_views(plan: &JanitorPlanText, style: CliStyle) -> String { + let listed: Vec<&OldNameViewText> = plan + .old_name_drops + .iter() + .chain(&plan.old_name_live) + .chain(&plan.old_name_missing) + .collect(); + if listed.is_empty() { + return String::new(); + } + let mut text: String = format!( + "\n{} {}\n", + style.success("Old name views"), + style.muted(&format!("({})", listed.len())) + ); + for (index, view) in listed.iter().enumerate() { + let last: bool = index + 1 == listed.len(); + text.push_str(&format!( + "{} {} {}\n", + tree_connector(style, last), + style.object_name(&view.name), + style.muted(&format!("-> model:{}", view.destination_model)) + )); + let (label, detail) = old_name_detail(view); + let indent: String = if last { + TREE_BLANK.to_owned() + } else { + style.muted(TREE_CONTINUATION) + }; + text.push_str(&format!( + "{indent}{} {} {detail}\n", + tree_connector(style, true), + style.muted(label) + )); + } + text +} + +/// A titled list of ` name` or ` name detail` lines, or nothing when empty. +pub(crate) fn listing(style: CliStyle, title: &str, items: &[(String, Option)]) -> String { + if items.is_empty() { + return String::new(); + } + let mut text: String = format!("\n{}\n", style.success(title)); + for (name, detail) in items { + match detail { + Some(detail) => text.push_str(&format!( + " {} {}\n", + style.object_name(name), + style.muted(detail) + )), + None => text.push_str(&format!(" {}\n", style.object_name(name))), + } + } + text +} diff --git a/crates/sqlbuild-core/src/janitor_text/constants.rs b/crates/sqlbuild-core/src/janitor_text/constants.rs new file mode 100644 index 0000000000..d9ebfac56c --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/constants.rs @@ -0,0 +1,13 @@ +//! Labels and layout of `sqb janitor` output. + +pub(crate) const LABEL_WIDTH: usize = 22; +pub(crate) const TREE_BRANCH_GLYPH: &str = "\u{251c}\u{2500}\u{2500}"; +pub(crate) const TREE_LAST_GLYPH: &str = "\u{2514}\u{2500}\u{2500}"; +pub(crate) const TREE_CONTINUATION: &str = "\u{2502} "; +pub(crate) const TREE_BLANK: &str = " "; +pub(crate) const DEFAULT_ENVIRONMENT: &str = "default"; +pub(crate) const DROP_EARLY: &str = "early"; +pub(crate) const DROP_EXPIRED: &str = "expired"; +pub(crate) const DROP_MISSING: &str = "missing"; +pub(crate) const DISABLED_DETAIL: &str = "Janitor is opt-in. It previews stale warehouse objects \ +and asks before archiving or deleting anything."; diff --git a/crates/sqlbuild-core/src/janitor_text/errors.rs b/crates/sqlbuild-core/src/janitor_text/errors.rs new file mode 100644 index 0000000000..52fa273724 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/errors.rs @@ -0,0 +1,8 @@ +//! Invalid janitor settings. + +/// A CLI user error: its code and message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JanitorSettingsError { + pub code: &'static str, + pub message: &'static str, +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_completion_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_completion_text.rs new file mode 100644 index 0000000000..c6d4ef48b9 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_completion_text.rs @@ -0,0 +1,35 @@ +//! The summary of a completed janitor run. + +use crate::cli_text::main::format_count_noun::format_count_noun; +use crate::cli_text::models::CliStyle; +use crate::janitor_text::models::JanitorResultCounts; + +fn noun(count: usize, singular: &str) -> String { + format_count_noun(i64::try_from(count).unwrap_or(i64::MAX), singular, None) +} + +/// `[Archived N relations. ]Deleted M objects[ and pruned K direct state tables].` +#[must_use] +pub fn janitor_completion_text(counts: &JanitorResultCounts, style: CliStyle) -> String { + let archived: String = if counts.archived == 0 { + String::new() + } else { + format!("Archived {}. ", noun(counts.archived, "relation")) + }; + let objects: String = noun( + counts.deleted + + counts.deleted_archives + + counts.deleted_query_artifacts + + counts.dropped_old_name_views, + "object", + ); + let deleted: String = if counts.pruned_direct_state == 0 { + format!("Deleted {objects}.") + } else { + format!( + "Deleted {objects} and pruned {}.", + noun(counts.pruned_direct_state, "direct state table") + ) + }; + format!("{}\n", style.success(&format!("{archived}{deleted}"))) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_phrase.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_phrase.rs new file mode 100644 index 0000000000..cd5a014db4 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_phrase.rs @@ -0,0 +1,24 @@ +//! The exact phrase that confirms a janitor plan. + +use crate::janitor_text::_helpers::counts::{ + archive_prefix, environment_label, physical_deletion_count, +}; +use crate::janitor_text::models::JanitorPlanText; + +/// `[archive N and ]delete M objects|items from `. +#[must_use] +pub fn janitor_confirmation_phrase(plan: &JanitorPlanText) -> String { + let prefix: String = archive_prefix(plan); + let deletions: usize = physical_deletion_count(plan); + if plan.state_prunes.is_empty() { + return format!( + "{prefix}delete {deletions} objects from {}", + environment_label(plan) + ); + } + format!( + "{prefix}delete {} items from {}", + deletions + plan.state_prunes.len(), + environment_label(plan) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs new file mode 100644 index 0000000000..0db1e14284 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs @@ -0,0 +1,41 @@ +//! What janitor will do, and the prompt for its confirmation phrase. + +use crate::janitor_text::_helpers::counts::{ + archive_prefix, environment_label, physical_deletion_count, +}; +use crate::janitor_text::main::janitor_confirmation_phrase::janitor_confirmation_phrase; +use crate::janitor_text::models::JanitorPlanText; + +/// The summary, retention lines and `Type ... to continue: ` prompt. +#[must_use] +pub fn janitor_confirmation_prompt(plan: &JanitorPlanText) -> String { + let prefix: String = archive_prefix(plan); + let environment: &str = environment_label(plan); + let mut text: String = if plan.state_prunes.is_empty() { + let objects: usize = if plan.direct_mode { + physical_deletion_count(plan) + } else { + plan.candidates.len() + plan.query_artifacts.len() + }; + format!("Janitor will {prefix}delete {objects} objects from {environment}.\n") + } else { + let items: usize = physical_deletion_count(plan) + plan.state_prunes.len(); + format!("Janitor will {prefix}delete {items} items from {environment}.\n") + }; + if plan.retention_days == 0 { + text.push_str("Retention: disabled (0 days)\nAge metadata will not be checked.\n"); + } else { + text.push_str(&format!("Retention: {} days\n", plan.retention_days)); + } + if plan.direct_mode { + text.push_str(&format!( + "Archive retention: {} days\n", + plan.archive_retention_days + )); + } + text.push_str(&format!( + "\nType `{}` to continue: ", + janitor_confirmation_phrase(plan) + )); + text +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs new file mode 100644 index 0000000000..79fabfc34e --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs @@ -0,0 +1,17 @@ +//! Guidance when janitor is not enabled for the project. + +use crate::cli_text::models::CliStyle; +use crate::janitor_text::constants::DISABLED_DETAIL; + +/// How to enable janitor and preview cleanup. +#[must_use] +pub fn janitor_disabled_text(style: CliStyle) -> String { + format!( + "{}\n\n{}\n\nAdd this block to {}:\n\n[janitor]\nenabled = true\n\n\ + After enabling, run janitor again to preview cleanup:\n {}\n", + style.warning_strong("Janitor is disabled for this project."), + style.warning(DISABLED_DETAIL), + style.object_name("sqlbuild_project.toml"), + style.accent("sqb janitor") + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_plan_has_work.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_plan_has_work.rs new file mode 100644 index 0000000000..bb4e554b5c --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_plan_has_work.rs @@ -0,0 +1,15 @@ +//! Whether a janitor plan changes anything. + +use crate::janitor_text::models::JanitorPlanText; + +/// True when the plan deletes, archives, prunes or drops anything. +#[must_use] +pub fn janitor_plan_has_work(plan: &JanitorPlanText) -> bool { + (!plan.candidates.is_empty() && !plan.direct_mode) + || !plan.archive_candidates.is_empty() + || !plan.archive_deletions.is_empty() + || !plan.query_artifacts.is_empty() + || !plan.state_prunes.is_empty() + || !plan.old_name_drops.is_empty() + || !plan.old_name_missing.is_empty() +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_preview_complete_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_preview_complete_text.rs new file mode 100644 index 0000000000..01b7d1d684 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_preview_complete_text.rs @@ -0,0 +1,20 @@ +//! The close of an `--as` preview with an unambiguous nothing-changed state. + +use crate::cli_text::models::CliStyle; +use crate::janitor_text::_helpers::counts::connection_label; + +/// `Previewed janitor ... Nothing was changed.` and how to apply it. +#[must_use] +pub fn janitor_preview_complete_text( + as_target: &str, + active_target: Option<&str>, + style: CliStyle, +) -> String { + format!( + "{}\nRerun with `--target {as_target}` instead of `--as {as_target}` to apply it.\n", + style.success(&format!( + "Previewed janitor as target '{as_target}' through {}. Nothing was changed.", + connection_label(active_target) + )) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_preview_start_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_preview_start_text.rs new file mode 100644 index 0000000000..9d1b482da4 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_preview_start_text.rs @@ -0,0 +1,12 @@ +//! The announcement of an inspection-only `--as` preview. + +use crate::janitor_text::_helpers::counts::connection_label; + +/// `Previewing janitor as target ... (inspection only).` +#[must_use] +pub fn janitor_preview_start_text(as_target: &str, active_target: Option<&str>) -> String { + format!( + "Previewing janitor as target '{as_target}' through {} (inspection only).\n", + connection_label(active_target) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/mod.rs b/crates/sqlbuild-core/src/janitor_text/main/mod.rs new file mode 100644 index 0000000000..d73e81e797 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/mod.rs @@ -0,0 +1,9 @@ +pub mod janitor_completion_text; +pub mod janitor_confirmation_phrase; +pub mod janitor_confirmation_prompt; +pub mod janitor_disabled_text; +pub mod janitor_plan_has_work; +pub mod janitor_preview_complete_text; +pub mod janitor_preview_start_text; +pub mod render_janitor_plan; +pub mod validate_janitor_settings; diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs new file mode 100644 index 0000000000..f9191b40ed --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs @@ -0,0 +1,68 @@ +//! The janitor plan preview. + +use crate::cli_text::models::CliStyle; +use crate::janitor_text::_helpers::counts::environment_label; +use crate::janitor_text::_helpers::plan_sections::{ + archives, blocked, listing, old_name_views, summary, +}; +use crate::janitor_text::models::JanitorPlanText; + +/// Every section of the preview, ending with a blank line. +#[must_use] +pub fn render_janitor_plan(plan: &JanitorPlanText, style: CliStyle) -> String { + let mut text: String = format!( + "{} {}\n\n", + style.title("Janitor preview"), + style.object_name(environment_label(plan)) + ); + text.push_str(&summary(plan, style)); + text.push_str(&blocked(plan, style)); + let skipped_schemas: Vec<(String, Option)> = plan + .skipped_schemas + .iter() + .map(|schema| { + ( + schema.name.clone(), + Some(format!( + "contains active source {}", + schema.source_names.join(", ") + )), + ) + }) + .collect(); + text.push_str(&listing(style, "Skipped schemas", &skipped_schemas)); + if !plan.direct_mode { + let candidates: Vec<(String, Option)> = plan + .candidates + .iter() + .map(|name| (name.clone(), None)) + .collect(); + text.push_str(&listing(style, "Eligible objects", &candidates)); + } + text.push_str(&archives(plan, style)); + text.push_str(&old_name_views(plan, style)); + let artifacts: Vec<(String, Option)> = plan + .query_artifacts + .iter() + .map(|name| (name.clone(), None)) + .collect(); + text.push_str(&listing( + style, + "Eligible expired query artifacts", + &artifacts, + )); + let prunes: Vec<(String, Option)> = plan + .state_prunes + .iter() + .map(|(name, versions)| (name.clone(), Some(format!("keep latest {versions}")))) + .collect(); + text.push_str(&listing(style, "Eligible direct state pruning", &prunes)); + let skipped: Vec<(String, Option)> = plan + .skipped_relations + .iter() + .map(|(name, reason)| (name.clone(), Some(reason.clone()))) + .collect(); + text.push_str(&listing(style, "Skipped objects", &skipped)); + text.push('\n'); + text +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/validate_janitor_settings.rs b/crates/sqlbuild-core/src/janitor_text/main/validate_janitor_settings.rs new file mode 100644 index 0000000000..051afabe82 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/validate_janitor_settings.rs @@ -0,0 +1,23 @@ +//! Validate effective janitor retention settings. + +use crate::janitor_text::errors::JanitorSettingsError; + +/// Reject negative retention or direct state history settings. +pub fn validate_janitor_settings( + retention_days: i64, + direct_state_history_versions: i64, +) -> Result<(), JanitorSettingsError> { + if retention_days < 0 { + return Err(JanitorSettingsError { + code: "C501", + message: "janitor --retention-days must be >= 0", + }); + } + if direct_state_history_versions < 0 { + return Err(JanitorSettingsError { + code: "C502", + message: "janitor --direct-state-history-versions must be >= 0", + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/janitor_text/mod.rs b/crates/sqlbuild-core/src/janitor_text/mod.rs new file mode 100644 index 0000000000..5fe11725d7 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/mod.rs @@ -0,0 +1,10 @@ +//! `sqb janitor` text: the plan preview, confirmation, previews, settings and completion. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/janitor_text/models.rs b/crates/sqlbuild-core/src/janitor_text/models.rs new file mode 100644 index 0000000000..b979149679 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/models.rs @@ -0,0 +1,83 @@ +//! Display facts of a janitor plan and its execution, read from the executor's models. + +/// A relation to archive: its name, archive name, age and expiry. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ArchiveCandidateText { + pub name: String, + pub archive_name: String, + /// Whole days old at planning time, or none when unknown. + pub age_days: Option, + /// Whether the archive expires no later than it is created. + pub expires_now: bool, + /// The expiry as `YYYY-MM-DD HH:MM:SS`. + pub expires_at: String, +} + +/// An archived relation to delete or retain. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ArchivedRelationText { + pub name: String, + /// Whether this run archives it. + pub archived_in_run: bool, + /// The archive time as `YYYY-MM-DD HH:MM:SS`. + pub archived_at: String, + pub age_days: Option, + pub expires_at: String, +} + +/// A compatibility view at a model's old name. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct OldNameViewText { + pub name: String, + pub destination_model: String, + /// The expiry as `YYYY-MM-DD`, or none. + pub expires_on: Option, + pub drop_reason: Option, + pub occupied: bool, + pub claimed_by: Option, +} + +/// A schema skipped or blocked by active sources. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SourceSchemaText { + pub name: String, + pub source_names: Vec, + pub suppressed_candidates: Vec, + pub suppressed_archive_deletions: Vec, +} + +/// One janitor plan as its preview, confirmation and work check read it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct JanitorPlanText { + pub target_name: Option, + pub retention_days: i64, + pub direct_mode: bool, + pub archive_retention_days: i64, + pub age_metadata_supported: bool, + pub scanned_schema_count: i64, + pub candidates: Vec, + pub archive_candidates: Vec, + pub archive_deletions: Vec, + pub retained_archives: Vec, + pub old_name_drops: Vec, + pub old_name_live: Vec, + pub old_name_missing: Vec, + pub query_artifacts: Vec, + /// Direct state tables to prune and the versions each keeps. + pub state_prunes: Vec<(String, i64)>, + /// Skipped relations and why. + pub skipped_relations: Vec<(String, String)>, + pub skipped_schemas: Vec, + pub blocked_schemas: Vec, +} + +/// How many items one janitor run changed. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct JanitorResultCounts { + pub archived: usize, + pub deleted: usize, + pub deleted_archives: usize, + pub deleted_query_artifacts: usize, + pub dropped_old_name_views: usize, + pub pruned_direct_state: usize, +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests.rs b/crates/sqlbuild-core/src/janitor_text/tests.rs new file mode 100644 index 0000000000..67b4ac8a4c --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_janitor_text.rs"] +mod janitor_text; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs new file mode 100644 index 0000000000..f304628d0a --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -0,0 +1,20 @@ +use crate::janitor_text::models::{ArchiveCandidateText, ArchivedRelationText, JanitorPlanText}; + +/// A direct-mode plan for `dev` that archives one relation and deletes one archive. +pub(super) fn direct_archive_plan() -> JanitorPlanText { + JanitorPlanText { + target_name: Some("dev".to_owned()), + retention_days: 14, + direct_mode: true, + archive_retention_days: 14, + archive_candidates: vec![ArchiveCandidateText { + name: "dev.old_orders".to_owned(), + ..ArchiveCandidateText::default() + }], + archive_deletions: vec![ArchivedRelationText { + name: "dev.old_products".to_owned(), + ..ArchivedRelationText::default() + }], + ..JanitorPlanText::default() + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_text.rs new file mode 100644 index 0000000000..b08556d7a4 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_text.rs @@ -0,0 +1,102 @@ +use crate::cli_text::models::CliStyle; +use crate::janitor_text::main::janitor_confirmation_prompt::janitor_confirmation_prompt; +use crate::janitor_text::main::janitor_disabled_text::janitor_disabled_text; +use crate::janitor_text::main::validate_janitor_settings::validate_janitor_settings; +use crate::janitor_text::models::JanitorPlanText; +use crate::janitor_text::tests::helpers::direct_archive_plan; +use crate::janitor_text::tests::test_types::{ + ConfirmationTestCase, DisabledTestCase, SettingsTestCase, +}; + +#[test] +fn given_plan_when_prompting_then_phrase_counts_every_action() { + let test_cases = [ + ConfirmationTestCase { + description: "a direct archive plan names archive and delete counts", + plan: direct_archive_plan(), + expected_prompt: "Janitor will archive 1 and delete 1 objects from dev.\n\ + Retention: 14 days\nArchive retention: 14 days\n\n\ + Type `archive 1 and delete 1 objects from dev` to continue: ", + }, + ConfirmationTestCase { + description: "state pruning counts items and zero retention skips age checks", + plan: JanitorPlanText { + target_name: None, + retention_days: 0, + state_prunes: vec![("main._sqlbuild_fingerprints".to_owned(), 2)], + ..direct_archive_plan() + }, + expected_prompt: "Janitor will archive 1 and delete 2 items from default.\n\ + Retention: disabled (0 days)\nAge metadata will not be checked.\n\ + Archive retention: 14 days\n\n\ + Type `archive 1 and delete 2 items from default` to continue: ", + }, + ]; + + for test_case in test_cases { + assert_eq!( + janitor_confirmation_prompt(&test_case.plan), + test_case.expected_prompt, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_settings_when_validating_then_negative_values_fail() { + let test_cases = [ + SettingsTestCase { + description: "zero values are valid", + retention_days: 0, + history_versions: 0, + expected_code: None, + }, + SettingsTestCase { + description: "negative retention fails first", + retention_days: -1, + history_versions: -1, + expected_code: Some("C501"), + }, + SettingsTestCase { + description: "negative history fails", + retention_days: 3, + history_versions: -1, + expected_code: Some("C502"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + validate_janitor_settings(test_case.retention_days, test_case.history_versions) + .err() + .map(|error| error.code), + test_case.expected_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_disabled_janitor_when_rendering_then_guidance_matches() { + let test_cases = [DisabledTestCase { + description: "preserves disabled no-color output", + use_color: false, + expected_text: "Janitor is disabled for this project.\n\n\ + Janitor is opt-in. It previews stale warehouse objects and asks before archiving or \ + deleting anything.\n\nAdd this block to sqlbuild_project.toml:\n\n[janitor]\n\ + enabled = true\n\nAfter enabling, run janitor again to preview cleanup:\n sqb janitor\n", + }]; + + for test_case in test_cases { + assert_eq!( + janitor_disabled_text(CliStyle { + use_color: test_case.use_color + }), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs new file mode 100644 index 0000000000..6a91f57366 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs @@ -0,0 +1,20 @@ +use crate::janitor_text::models::JanitorPlanText; + +pub(super) struct ConfirmationTestCase { + pub(super) description: &'static str, + pub(super) plan: JanitorPlanText, + pub(super) expected_prompt: &'static str, +} + +pub(super) struct SettingsTestCase { + pub(super) description: &'static str, + pub(super) retention_days: i64, + pub(super) history_versions: i64, + pub(super) expected_code: Option<&'static str>, +} + +pub(super) struct DisabledTestCase { + pub(super) description: &'static str, + pub(super) use_color: bool, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index c9a089b7be..5afcac9c29 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -4,10 +4,16 @@ pub mod cli_text; pub mod constants; +pub mod cost_report; +pub mod debug_report; +pub mod freshness_text; +pub mod integration_results; +pub mod janitor_text; pub mod json; pub mod models; pub mod panics; pub mod posix_path; +pub mod progress; pub mod terminal_events; pub mod text; diff --git a/crates/sqlbuild-core/src/progress/_helpers/mod.rs b/crates/sqlbuild-core/src/progress/_helpers/mod.rs new file mode 100644 index 0000000000..e40819d63d --- /dev/null +++ b/crates/sqlbuild-core/src/progress/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod operations; +pub(crate) mod resources; +pub(crate) mod statements; diff --git a/crates/sqlbuild-core/src/progress/_helpers/operations.rs b/crates/sqlbuild-core/src/progress/_helpers/operations.rs new file mode 100644 index 0000000000..0b74d456f8 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/_helpers/operations.rs @@ -0,0 +1,166 @@ +//! Operation start and terminal rows, and scheduled retries. + +use crate::cli_text::models::CliStyle; +use crate::progress::constants::{ + FAILED_SUFFIX, FRESHNESS_METADATA_OPERATION, FRESHNESS_QUERY_OPERATION, NESTED_PREFIX, + TTY_RESOURCE_OWNER_OPERATIONS, TTY_TRANSIENT_OWNER_OPERATIONS, VISIBLE_OPERATION_LABELS, +}; +use crate::progress::models::{ProgressEvent, ProgressProjection}; + +fn operation_label(operation_name: &str) -> Option<&'static str> { + VISIBLE_OPERATION_LABELS + .iter() + .find(|(name, _)| *name == operation_name) + .map(|(_, label)| *label) +} + +/// ` {attempt}` when the operation metadata carries an attempt number. +fn operation_attempt_label(event: &ProgressEvent) -> String { + event + .payload + .metadata_attempt_number + .map(|attempt| format!(" {attempt}")) + .unwrap_or_default() +} + +fn nested_prefix(event: &ProgressEvent) -> &'static str { + if event.resource_attempt_id.is_some() { + NESTED_PREFIX + } else { + "" + } +} + +/// Whether a live terminal owner shows this operation instead of a persistent row. +fn owned_by_live_row(projection: &ProgressProjection, event: &ProgressEvent, name: &str) -> bool { + projection.settings.is_tty + && if event.resource_attempt_id.is_none() { + TTY_TRANSIENT_OWNER_OPERATIONS.contains(&name) + } else { + TTY_RESOURCE_OWNER_OPERATIONS.contains(&name) + } +} + +/// Render the start row of a visible operation. +pub(crate) fn consume_operation_start( + mut projection: ProgressProjection, + event: &ProgressEvent, +) -> (ProgressProjection, Option) { + let (Some(operation_id), Some(operation_name)) = + (&event.operation_id, &event.payload.operation_name) + else { + return (projection, None); + }; + let Some(label) = operation_label(operation_name) else { + return (projection, None); + }; + if owned_by_live_row(&projection, event, operation_name) { + return (projection, None); + } + projection + .operation_starts + .insert(operation_id.clone(), event.clone()); + if event.resource_attempt_id.is_some() && !projection.settings.debug { + projection.quiet_operation_ids.insert(operation_id.clone()); + return (projection, None); + } + let status: String = CliStyle { + use_color: projection.settings.use_color, + } + .status("START"); + let line: String = format!( + "{}{label}{} {status}", + nested_prefix(event), + operation_attempt_label(event) + ); + (projection, Some(line)) +} + +/// Render the terminal row of a started visible operation. +pub(crate) fn consume_operation_terminal( + mut projection: ProgressProjection, + event: &ProgressEvent, +) -> (ProgressProjection, Option) { + let Some(operation_id) = &event.operation_id else { + return (projection, None); + }; + let Some(start) = projection.operation_starts.remove(operation_id) else { + return (projection, None); + }; + let failed: bool = event.event_type.ends_with(FAILED_SUFFIX); + if projection.quiet_operation_ids.remove(operation_id) && !failed { + return (projection, None); + } + let Some(label) = start + .payload + .operation_name + .as_deref() + .and_then(operation_label) + else { + return (projection, None); + }; + let status: String = CliStyle { + use_color: projection.settings.use_color, + } + .status(if failed { "FAIL" } else { "OK" }); + let elapsed: String = event + .payload + .duration_ms + .map(|duration| format!(" ({:.2}s)", duration / 1000.0)) + .unwrap_or_default(); + let detail: String = operation_terminal_detail( + start.payload.operation_name.as_deref().unwrap_or_default(), + event, + ); + let line: String = format!( + "{}{label}{} {status}{detail}{elapsed}", + nested_prefix(event), + operation_attempt_label(&start) + ); + (projection, Some(line)) +} + +/// The freshness summary a freshness observation terminal adds. +fn operation_terminal_detail(operation_name: &str, event: &ProgressEvent) -> String { + let failed: bool = event.event_type.ends_with(FAILED_SUFFIX); + if operation_name == FRESHNESS_QUERY_OPERATION { + return if failed { + " (source freshness unknown)".to_owned() + } else { + String::new() + }; + } + if operation_name != FRESHNESS_METADATA_OPERATION { + return String::new(); + } + let Some(item_count) = event.payload.metadata_item_count else { + return String::new(); + }; + let noun: &str = if item_count == 1 { "source" } else { "sources" }; + if failed { + let reason: String = event + .payload + .error_type + .as_ref() + .map(|error_type| format!(": {error_type}")) + .unwrap_or_default(); + return format!(" ({item_count} {noun} unknown, metadata lookup failed{reason})"); + } + let unknown: i64 = event.payload.metadata_unknown_count.unwrap_or(0); + format!(" ({item_count} {noun}, {unknown} unknown)") +} + +/// Render one scheduled retry. +pub(crate) fn retry_scheduled_line(event: &ProgressEvent) -> Option { + let (Some(failed_attempt), Some(next_attempt), Some(delay_ms)) = ( + event.payload.failed_attempt_number, + event.payload.next_attempt_number, + event.payload.delay_ms, + ) else { + return None; + }; + let delay_seconds: f64 = delay_ms / 1000.0; + Some(format!( + "{NESTED_PREFIX}retry {failed_attempt}->{next_attempt} in {delay_seconds:.2}s" + )) +} diff --git a/crates/sqlbuild-core/src/progress/_helpers/resources.rs b/crates/sqlbuild-core/src/progress/_helpers/resources.rs new file mode 100644 index 0000000000..f1ca1aeaca --- /dev/null +++ b/crates/sqlbuild-core/src/progress/_helpers/resources.rs @@ -0,0 +1,250 @@ +//! Resource attempt rows: starts, terminals, and the terminals a result callback claims. + +use std::collections::VecDeque; + +use crate::cli_text::models::CliStyle; +use crate::progress::constants::{ + FAILED_SUFFIX, KIND_WIDTH, SCENARIO_RESOURCE_PREFIX, SKIPPED_SUFFIX, +}; +use crate::progress::models::{ProgressEvent, ProgressProjection}; +use crate::terminal_events::models::TerminalRequest; + +/// The resource name, prefixed by its scenario when the resource runs inside one. +fn display_resource_name(resource_id: Option<&str>, resource_name: &str) -> String { + let Some(scenario) = resource_id.and_then(|id| id.strip_prefix(SCENARIO_RESOURCE_PREFIX)) + else { + return resource_name.to_owned(); + }; + let scenario_name: &str = scenario.split(':').next().unwrap_or(scenario); + format!("{scenario_name}/{resource_name}") +} + +/// Remember an attempt, keeping the position of its first sighting. +fn store_attempt( + mut projection: ProgressProjection, + attempt_id: &str, + event: &ProgressEvent, +) -> ProgressProjection { + if projection + .attempts_by_id + .insert(attempt_id.to_owned(), event.clone()) + .is_none() + { + projection.attempt_order.push(attempt_id.to_owned()); + } + projection +} + +fn remember_resource( + mut projection: ProgressProjection, + resource_name: &str, + resource_id: &str, +) -> ProgressProjection { + projection + .resource_ids_by_name + .entry(resource_name.to_owned()) + .or_default() + .insert(resource_id.to_owned()); + projection +} + +/// Render the start row of one resource attempt. +pub(crate) fn consume_resource_start( + projection: ProgressProjection, + event: &ProgressEvent, +) -> (ProgressProjection, Vec) { + let (Some(attempt_id), Some(resource_id), Some(resource_name), Some(resource_kind)) = ( + &event.resource_attempt_id, + &event.resource_id, + &event.payload.resource_name, + &event.payload.resource_kind, + ) else { + return (projection, Vec::new()); + }; + let mut projection: ProgressProjection = + remember_resource(projection, resource_name, resource_id); + projection + .resource_names + .insert(resource_id.clone(), resource_name.clone()); + let (projection, mut lines) = render_prior_attempts(projection, (resource_id, resource_name)); + let projection: ProgressProjection = store_attempt(projection, attempt_id, event); + if projection.settings.is_tty + && (projection.enriched_resource_names.contains(resource_name) + || !projection.settings.debug) + { + return (projection, lines); + } + let counter: String = match projection.resource_ordinals.get(resource_name) { + Some(ordinal) if projection.resource_total > 0 => { + format!("{ordinal}/{} ", projection.resource_total) + } + _ => String::new(), + }; + let status: String = CliStyle { + use_color: projection.settings.use_color, + } + .status("START"); + let display_name: String = display_resource_name(Some(resource_id), resource_name); + lines.push(format!( + " {counter}{resource_kind: (ProgressProjection, Vec) { + let (Some(attempt_id), Some(resource_id), Some(resource_name)) = ( + &event.resource_attempt_id, + &event.resource_id, + &event.payload.resource_name, + ) else { + return (projection, Vec::new()); + }; + let projection: ProgressProjection = remember_resource(projection, resource_name, resource_id); + let mut projection: ProgressProjection = store_attempt(projection, attempt_id, event); + projection + .pending_terminal_ids_by_resource + .entry(resource_id.clone()) + .or_default() + .push_back(attempt_id.clone()); + if projection.enriched_resource_names.contains(resource_name) { + return (projection, Vec::new()); + } + let (projection, line) = render_resource_terminal(projection, event, resource_name); + (projection, line.into_iter().collect()) +} + +/// Render the terminal row of one attempt once. +pub(crate) fn render_resource_terminal( + mut projection: ProgressProjection, + event: &ProgressEvent, + resource_name: &str, +) -> (ProgressProjection, Option) { + let Some(attempt_id) = &event.resource_attempt_id else { + return (projection, None); + }; + if !projection.rendered_terminal_ids.insert(attempt_id.clone()) { + return (projection, None); + } + let Some(resource_kind) = &event.payload.resource_kind else { + return (projection, None); + }; + let status_text: &str = if event.event_type.ends_with(FAILED_SUFFIX) { + "FAIL" + } else if event.event_type.ends_with(SKIPPED_SUFFIX) { + "SKIP" + } else { + "OK" + }; + let status: String = CliStyle { + use_color: projection.settings.use_color, + } + .status(status_text); + let elapsed: String = event + .payload + .duration_ms + .map(|duration| format!(" {:.2}s", duration / 1000.0)) + .unwrap_or_default(); + let display_name: String = display_resource_name(event.resource_id.as_deref(), resource_name); + let line: String = format!(" {resource_kind: (ProgressProjection, Vec) { + let mut projection: ProgressProjection = projection; + let mut lines: Vec = Vec::new(); + for event in events { + let (rendered, line) = render_resource_terminal(projection, event, resource_name); + projection = rendered; + lines.extend(line); + } + (projection, lines) +} + +/// Render every unclaimed terminal of earlier attempts before a new attempt starts. +fn render_prior_attempts( + mut projection: ProgressProjection, + (resource_id, resource_name): (&str, &str), +) -> (ProgressProjection, Vec) { + let pending: VecDeque = projection + .pending_terminal_ids_by_resource + .remove(resource_id) + .unwrap_or_default(); + let events: Vec = pending + .iter() + .filter_map(|attempt_id| projection.attempts_by_id.get(attempt_id).cloned()) + .collect(); + render_resource_terminals(projection, &events, resource_name) +} + +/// The resource identifier a request names, or the only one seen for its name. +fn resolved_resource_id( + projection: &ProgressProjection, + request: &TerminalRequest, +) -> Option { + if let Some(resource_id) = request.resource_id.as_ref().filter(|id| !id.is_empty()) { + return Some(resource_id.clone()); + } + let resource_ids = projection + .resource_ids_by_name + .get(&request.resource_name)?; + if resource_ids.len() != 1 { + return None; + } + resource_ids.iter().next().cloned() +} + +/// The unrendered pending terminals of the requested resource, and the one the request selects. +pub(crate) fn resolve_resource_terminal( + projection: &ProgressProjection, + request: &TerminalRequest, +) -> Option<(String, Vec, usize)> { + let resource_id: String = resolved_resource_id(projection, request)?; + let candidates: Vec = projection + .pending_terminal_ids_by_resource + .get(&resource_id) + .into_iter() + .flatten() + .filter(|attempt_id| !projection.rendered_terminal_ids.contains(*attempt_id)) + .filter_map(|attempt_id| projection.attempts_by_id.get(attempt_id).cloned()) + .collect(); + let target: usize = if let Some(attempt_id) = &request.resource_attempt_id { + candidates + .iter() + .position(|event| event.resource_attempt_id.as_ref() == Some(attempt_id))? + } else if let Some(attempt_number) = request.attempt_number { + candidates + .iter() + .rposition(|event| event.payload.attempt_number == Some(attempt_number))? + } else { + candidates.len().checked_sub(1)? + }; + Some((resource_id, candidates, target)) +} + +/// Drop pending terminals up to and including the claimed one. +pub(crate) fn remove_claimed_terminals( + mut projection: ProgressProjection, + resource_id: &str, + target_attempt_id: &str, +) -> ProgressProjection { + if let Some(pending) = projection + .pending_terminal_ids_by_resource + .get_mut(resource_id) + { + while let Some(attempt_id) = pending.pop_front() { + if attempt_id == target_attempt_id { + break; + } + } + } + projection +} diff --git a/crates/sqlbuild-core/src/progress/_helpers/statements.rs b/crates/sqlbuild-core/src/progress/_helpers/statements.rs new file mode 100644 index 0000000000..76e95d91c5 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/_helpers/statements.rs @@ -0,0 +1,144 @@ +//! Warehouse statement rows of resource attempts: debug starts, heartbeats and failures. + +use crate::progress::constants::{NESTED_PREFIX, STATEMENT_FAILED, UNKNOWN_RESOURCE_NAME}; +use crate::progress::models::{ProgressEvent, ProgressProjection}; + +/// `model=... phase=... kind=...` for one statement. +fn statement_context(projection: &ProgressProjection, event: &ProgressEvent) -> String { + let resource_name: &str = projection + .resource_names + .get(event.resource_id.as_deref().unwrap_or_default()) + .map_or(UNKNOWN_RESOURCE_NAME, String::as_str); + let operation: Option<&ProgressEvent> = projection + .operation_starts + .get(event.operation_id.as_deref().unwrap_or_default()); + let phase: &str = operation + .and_then(|start| { + start + .payload + .phase + .as_deref() + .or(start.payload.operation_name.as_deref()) + }) + .unwrap_or(&event.payload.intent); + format!( + "model={resource_name} phase={phase} kind={}", + event.payload.statement_kind + ) +} + +/// The query identifier the event carries, or the one its statement submitted. +fn statement_query_id(projection: &ProgressProjection, event: &ProgressEvent) -> Option { + if let Some(query_id) = &event.payload.query_id { + return Some(query_id.clone()); + } + projection + .statement_query_ids + .get(event.statement_id.as_deref()?) + .cloned() +} + +fn query_suffix(query_id: Option<&String>) -> String { + query_id + .map(|query_id| format!(" query_id={query_id}")) + .unwrap_or_default() +} + +/// Track a statement of a known resource, rendering its start in debug mode. +pub(crate) fn consume_statement_start( + mut projection: ProgressProjection, + event: &ProgressEvent, +) -> (ProgressProjection, Option) { + let Some(statement_id) = &event.statement_id else { + return (projection, None); + }; + if !event + .resource_id + .as_ref() + .is_some_and(|resource_id| projection.resource_names.contains_key(resource_id)) + { + return (projection, None); + } + projection + .statement_starts + .insert(statement_id.clone(), event.clone()); + if !projection.settings.debug { + return (projection, None); + } + let context: String = statement_context(&projection, event); + ( + projection, + Some(format!("{NESTED_PREFIX}statement {context} START")), + ) +} + +/// Remember a submitted query identifier, rendering it in debug mode. +pub(crate) fn consume_statement_submitted( + mut projection: ProgressProjection, + event: &ProgressEvent, +) -> (ProgressProjection, Option) { + let (Some(statement_id), Some(query_id)) = (&event.statement_id, &event.payload.query_id) + else { + return (projection, None); + }; + let Some(start) = projection.statement_starts.get(statement_id) else { + return (projection, None); + }; + if projection.statement_query_ids.get(statement_id) == Some(query_id) { + return (projection, None); + } + let context: String = statement_context(&projection, start); + projection + .statement_query_ids + .insert(statement_id.clone(), query_id.clone()); + if !projection.settings.debug { + return (projection, None); + } + ( + projection, + Some(format!( + "{NESTED_PREFIX}statement {context} query_id={query_id}" + )), + ) +} + +/// Render a long-running statement heartbeat. +pub(crate) fn statement_heartbeat_line( + projection: &ProgressProjection, + event: &ProgressEvent, +) -> Option { + if !event + .statement_id + .as_ref() + .is_some_and(|statement_id| projection.statement_starts.contains_key(statement_id)) + { + return None; + } + let duration: f64 = event.payload.duration_ms?; + let suffix: String = query_suffix(statement_query_id(projection, event).as_ref()); + let context: String = statement_context(projection, event); + Some(format!( + "{NESTED_PREFIX}statement {context} RUNNING {:.1}s{suffix}", + duration / 1000.0 + )) +} + +/// Forget a finished statement, rendering it when it failed. +pub(crate) fn consume_statement_terminal( + mut projection: ProgressProjection, + event: &ProgressEvent, +) -> (ProgressProjection, Option) { + let Some(statement_id) = &event.statement_id else { + return (projection, None); + }; + let Some(start) = projection.statement_starts.remove(statement_id) else { + return (projection, None); + }; + let line: Option = (event.event_type == STATEMENT_FAILED).then(|| { + let suffix: String = query_suffix(statement_query_id(&projection, event).as_ref()); + let context: String = statement_context(&projection, &start); + format!("{NESTED_PREFIX}statement {context} FAIL{suffix}") + }); + projection.statement_query_ids.remove(statement_id); + (projection, line) +} diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs new file mode 100644 index 0000000000..a715a802dc --- /dev/null +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -0,0 +1,104 @@ +//! Lifecycle event types, operation labels and layout of CLI progress lines. + +/// Operations with visible progress lines, and the label each renders with. +pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 31] = [ + ("clone_finalization", "Clone finalization"), + ("discovery_declaration_parse", "Declaration parsing"), + ("discovery_filesystem_walk", "Filesystem discovery"), + ("discovery_project_assembly", "Project assembly"), + ("discovery_python_import", "Python discovery"), + ("external_manifest_discovery", "External manifest discovery"), + ("project_discovery", "Project discovery"), + ("project_compile", "Project compile"), + ("sql_test_planning", "SQL test planning"), + ("dbt_command", "dbt command"), + ("external_source_load", "loader callable"), + ("ingestr_command", "ingestr subprocess"), + ("managed_source_load", "loader callable"), + ("python_asset", "Python asset attempt"), + ("python_check", "Python check attempt"), + ("python_hook", "Python hook"), + ("python_task", "Python task attempt"), + ("sql_hook", "SQL hook"), + ("audit_evaluation", "Audit evaluation"), + ("sql_test_setup", "SQL test setup"), + ("sql_test_assertion", "SQL test assertion"), + ( + "source_freshness_metadata_observation", + "Freshness metadata", + ), + ("source_freshness_query_observation", "Freshness query"), + ("source_freshness_publication", "Publish freshness state"), + ("retention_inspection", "Inspect retention"), + ("retention_application", "Apply retention"), + ("table_type_inspection", "Inspect table type"), + ("table_type_conversion", "Convert table type"), + ("schema_synchronization", "Synchronize schema"), + ("staging_creation", "Create staging relation"), + ("relation_promotion", "Promote relation"), +]; +/// Operations a live transient owner shows on a terminal when no resource attempt owns them. +pub(crate) const TTY_TRANSIENT_OWNER_OPERATIONS: [&str; 18] = [ + "dbt_command", + "discovery_declaration_parse", + "discovery_filesystem_walk", + "discovery_project_assembly", + "discovery_python_import", + "external_manifest_discovery", + "external_source_load", + "ingestr_command", + "managed_source_load", + "project_compile", + "project_discovery", + "sql_test_planning", + "python_asset", + "python_check", + "python_hook", + "python_task", + "sql_hook", + "sql_test_setup", +]; +/// Resource-scoped operations a live result owner shows on a terminal. +pub(crate) const TTY_RESOURCE_OWNER_OPERATIONS: [&str; 2] = + ["audit_evaluation", "sql_test_assertion"]; +pub(crate) const FRESHNESS_METADATA_OPERATION: &str = "source_freshness_metadata_observation"; +pub(crate) const FRESHNESS_QUERY_OPERATION: &str = "source_freshness_query_observation"; +pub(crate) const RESOURCE_START: &str = "resource_attempt_started"; +pub(crate) const SCENARIO_RESOURCE_PREFIX: &str = "scenario:"; +pub(crate) const OPERATION_START: &str = "operation_started"; +pub(crate) const RETRY_SCHEDULED: &str = "retry_scheduled"; +pub(crate) const STATEMENT_START: &str = "statement_started"; +pub(crate) const STATEMENT_SUBMITTED: &str = "statement_submitted"; +pub(crate) const STATEMENT_HEARTBEAT: &str = "statement_heartbeat"; +pub(crate) const STATEMENT_COMPLETED: &str = "statement_completed"; +pub(crate) const STATEMENT_FAILED: &str = "statement_failed"; +pub(crate) const FAILED_SUFFIX: &str = "failed"; +pub(crate) const SKIPPED_SUFFIX: &str = "skipped"; +/// Width of the resource kind column of resource rows. +pub(crate) const KIND_WIDTH: usize = 10; +/// Indent of rows that belong to a resource attempt. +pub(crate) const NESTED_PREFIX: &str = " "; +pub(crate) const UNKNOWN_RESOURCE_NAME: &str = "warehouse"; +/// Messages that complete a planning phase rather than update it. +pub(crate) const PLANNING_COMPLETION_PREFIXES: [&str; 14] = [ + "Built ", + "Checked ", + "Compiled ", + "Evaluated ", + "Applied ", + "Finalized ", + "Generated ", + "Inspected ", + "Loaded ", + "Planned ", + "Recorded ", + "Refreshed ", + "Resolved ", + "Cloned ", +]; +pub(crate) const ASSERTION_PREFIX: &str = "assertion "; +pub(crate) const PASS_OUTCOME: &str = "pass"; +pub(crate) const ERROR_OUTCOME: &str = "error"; +/// Inspection SQL longer than this many characters is cut. +pub(crate) const SQL_PREVIEW_LENGTH: usize = 160; +pub(crate) const EXECUTION_TITLE: &str = "Execution"; diff --git a/crates/sqlbuild-core/src/progress/main/claim_progress_terminal.rs b/crates/sqlbuild-core/src/progress/main/claim_progress_terminal.rs new file mode 100644 index 0000000000..d962e74fd5 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/claim_progress_terminal.rs @@ -0,0 +1,30 @@ +//! Claim the terminal a final resource result reports on. + +use crate::progress::_helpers::resources::{ + remove_claimed_terminals, render_resource_terminals, resolve_resource_terminal, +}; +use crate::progress::models::ProgressProjection; +use crate::terminal_events::models::TerminalRequest; + +/// Claim the targeted or latest terminal; also returns the rows of earlier unclaimed attempts. +#[must_use] +pub fn claim_progress_terminal( + projection: ProgressProjection, + request: &TerminalRequest, +) -> (ProgressProjection, Vec, Option) { + let Some((resource_id, candidates, target)) = resolve_resource_terminal(&projection, request) + else { + return (projection, Vec::new(), None); + }; + let Some(target_attempt_id) = candidates[target].resource_attempt_id.clone() else { + return (projection, Vec::new(), None); + }; + let (mut projection, lines) = + render_resource_terminals(projection, &candidates[..target], &request.resource_name); + projection + .rendered_terminal_ids + .insert(target_attempt_id.clone()); + let projection: ProgressProjection = + remove_claimed_terminals(projection, &resource_id, &target_attempt_id); + (projection, lines, candidates[target].payload.duration_ms) +} diff --git a/crates/sqlbuild-core/src/progress/main/close_progress_projection.rs b/crates/sqlbuild-core/src/progress/main/close_progress_projection.rs new file mode 100644 index 0000000000..f2f46a6fff --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/close_progress_projection.rs @@ -0,0 +1,30 @@ +//! Render the resource terminals no result callback claimed. + +use crate::progress::_helpers::resources::render_resource_terminal; +use crate::progress::models::{ProgressEvent, ProgressProjection}; +use crate::terminal_events::constants::RESOURCE_TERMINALS; + +/// The rows of every unrendered resource terminal, in first-seen attempt order. +#[must_use] +pub fn close_progress_projection( + projection: ProgressProjection, +) -> (ProgressProjection, Vec) { + let terminals: Vec = projection + .attempt_order + .iter() + .filter_map(|attempt_id| projection.attempts_by_id.get(attempt_id)) + .filter(|event| RESOURCE_TERMINALS.contains(&event.event_type.as_str())) + .cloned() + .collect(); + let mut projection: ProgressProjection = projection; + let mut lines: Vec = Vec::new(); + for event in terminals { + let Some(resource_name) = event.payload.resource_name.clone() else { + continue; + }; + let (rendered, line) = render_resource_terminal(projection, &event, &resource_name); + projection = rendered; + lines.extend(line); + } + (projection, lines) +} diff --git a/crates/sqlbuild-core/src/progress/main/connection_phase_line.rs b/crates/sqlbuild-core/src/progress/main/connection_phase_line.rs new file mode 100644 index 0000000000..64c09e1ee2 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/connection_phase_line.rs @@ -0,0 +1,31 @@ +//! The line that ends the warehouse connection phase. + +use crate::cli_text::main::format_phase_line::format_phase_line; +use crate::cli_text::models::CliStyle; + +/// `✓ Warehouse connected (0.12s)`, or the failure line with the elapsed time. +#[must_use] +pub fn connection_phase_line( + style: CliStyle, + ok: bool, + (adapter_name, elapsed_seconds): (&str, f64), +) -> String { + if ok { + return format_phase_line( + style, + true, + ( + "Warehouse connected", + &format!("{adapter_name} ({elapsed_seconds:.2}s)"), + ), + ); + } + format_phase_line( + style, + false, + ( + "Warehouse connection failed", + &format!("{adapter_name} (after {elapsed_seconds:.2}s)"), + ), + ) +} diff --git a/crates/sqlbuild-core/src/progress/main/connection_start_message.rs b/crates/sqlbuild-core/src/progress/main/connection_start_message.rs new file mode 100644 index 0000000000..8b48b98448 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/connection_start_message.rs @@ -0,0 +1,10 @@ +//! The message shown while warehouse connections open. + +/// `Connecting to ...`, with the connection count when there are several. +#[must_use] +pub fn connection_start_message(adapter_name: &str, connection_count: i64) -> String { + if connection_count <= 1 { + return format!("Connecting to {adapter_name}..."); + } + format!("Connecting to {adapter_name} ({connection_count} connections)...") +} diff --git a/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs b/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs new file mode 100644 index 0000000000..09c10fccd3 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs @@ -0,0 +1,54 @@ +//! Project one lifecycle fact onto persistent progress lines. + +use crate::progress::_helpers::operations::{ + consume_operation_start, consume_operation_terminal, retry_scheduled_line, +}; +use crate::progress::_helpers::resources::{consume_resource_start, consume_resource_terminal}; +use crate::progress::_helpers::statements::{ + consume_statement_start, consume_statement_submitted, consume_statement_terminal, + statement_heartbeat_line, +}; +use crate::progress::constants::{ + OPERATION_START, RESOURCE_START, RETRY_SCHEDULED, STATEMENT_COMPLETED, STATEMENT_FAILED, + STATEMENT_HEARTBEAT, STATEMENT_START, STATEMENT_SUBMITTED, +}; +use crate::progress::models::{ProgressEvent, ProgressProjection}; +use crate::terminal_events::constants::{OPERATION_TERMINALS, RESOURCE_TERMINALS}; + +/// Consume a fact once; also returns the lines to write, without trailing newlines. +#[must_use] +pub fn consume_progress_event( + mut projection: ProgressProjection, + event: &ProgressEvent, +) -> (ProgressProjection, Vec) { + if !projection.seen_event_ids.insert(event.event_id.clone()) { + return (projection, Vec::new()); + } + let event_type: &str = event.event_type.as_str(); + if event_type == RESOURCE_START { + return consume_resource_start(projection, event); + } + if RESOURCE_TERMINALS.contains(&event_type) { + return consume_resource_terminal(projection, event); + } + let (projection, line) = if event_type == OPERATION_START { + consume_operation_start(projection, event) + } else if OPERATION_TERMINALS.contains(&event_type) { + consume_operation_terminal(projection, event) + } else if event_type == RETRY_SCHEDULED { + let line: Option = retry_scheduled_line(event); + (projection, line) + } else if event_type == STATEMENT_START { + consume_statement_start(projection, event) + } else if event_type == STATEMENT_SUBMITTED { + consume_statement_submitted(projection, event) + } else if event_type == STATEMENT_HEARTBEAT { + let line: Option = statement_heartbeat_line(&projection, event); + (projection, line) + } else if event_type == STATEMENT_COMPLETED || event_type == STATEMENT_FAILED { + consume_statement_terminal(projection, event) + } else { + (projection, None) + }; + (projection, line.into_iter().collect()) +} diff --git a/crates/sqlbuild-core/src/progress/main/execution_header_text.rs b/crates/sqlbuild-core/src/progress/main/execution_header_text.rs new file mode 100644 index 0000000000..8e7891f337 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/execution_header_text.rs @@ -0,0 +1,19 @@ +//! The execution header block written before command progress. + +use crate::cli_text::main::format_surface_header::format_surface_header; +use crate::cli_text::models::CliStyle; +use crate::progress::constants::EXECUTION_TITLE; +use crate::progress::main::format_execution_header::format_execution_header; + +/// The styled `Execution` header with its context, followed by a blank line. +#[must_use] +pub fn execution_header_text( + style: CliStyle, + (command, target, concurrency): (&str, Option<&str>, i64), +) -> String { + let context: String = format_execution_header(command, target, concurrency); + format!( + "{}\n\n", + format_surface_header(style, EXECUTION_TITLE, &context) + ) +} diff --git a/crates/sqlbuild-core/src/progress/main/format_execution_header.rs b/crates/sqlbuild-core/src/progress/main/format_execution_header.rs new file mode 100644 index 0000000000..3849dda93b --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/format_execution_header.rs @@ -0,0 +1,12 @@ +//! The command and execution context of a progress header. + +/// `command (target: ..., concurrency: N)`. +#[must_use] +pub fn format_execution_header(command: &str, target: Option<&str>, concurrency: i64) -> String { + let mut context_parts: Vec = Vec::new(); + if let Some(target) = target { + context_parts.push(format!("target: {target}")); + } + context_parts.push(format!("concurrency: {concurrency}")); + format!("{command} ({})", context_parts.join(", ")) +} diff --git a/crates/sqlbuild-core/src/progress/main/format_expectation_detail.rs b/crates/sqlbuild-core/src/progress/main/format_expectation_detail.rs new file mode 100644 index 0000000000..fb9ed9533e --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/format_expectation_detail.rs @@ -0,0 +1,63 @@ +//! The failing row detail of one SQL expectation row. + +use crate::progress::constants::{ASSERTION_PREFIX, ERROR_OUTCOME, PASS_OUTCOME}; +use crate::progress::models::ExpectationStep; +use crate::progress::types::DifferenceSample; + +/// Python's rendering of an optional count: the number, or `None`. +fn optional_count(count: Option) -> String { + count.map_or_else(|| "None".to_owned(), |count| count.to_string()) +} + +fn sample_lines(label: &str, samples: &[DifferenceSample]) -> Vec { + let mut lines: Vec = Vec::new(); + for (index, sample) in samples.iter().enumerate() { + let values: Vec = sample + .iter() + .map(|(name, value)| format!("{name}={value}")) + .collect(); + lines.push(format!("{label} {}: {}", index + 1, values.join(", "))); + } + lines +} + +/// Row counts, samples and differing columns of a failing step; empty when it passed or errored. +#[must_use] +pub fn format_expectation_detail(step: &ExpectationStep) -> String { + if step.outcome == PASS_OUTCOME || step.outcome == ERROR_OUTCOME { + return String::new(); + } + if step.model_name.starts_with(ASSERTION_PREFIX) { + let row_label: &str = if step.actual_row_count == 1 { + "row" + } else { + "rows" + }; + return format!(" {} {row_label}", step.actual_row_count); + } + if step.unexpected_row_count.is_none() + && step.missing_row_count.is_none() + && step.unexpected_samples.is_empty() + && step.missing_samples.is_empty() + { + return format!(" {} mismatched", step.mismatched_row_count); + } + let mut parts: Vec = vec![ + format!("unexpected={}", optional_count(step.unexpected_row_count)), + format!("missing={}", optional_count(step.missing_row_count)), + ]; + if step.actual_row_count != step.expected_row_count { + parts.push(format!( + "row counts actual={} expected={}", + step.actual_row_count, step.expected_row_count + )); + } + parts.extend(sample_lines("unexpected sample", &step.unexpected_samples)); + parts.extend(sample_lines("missing sample", &step.missing_samples)); + for (name, actual, expected) in &step.column_differences { + parts.push(format!( + "differing column {name}: actual={actual}, expected={expected}" + )); + } + format!(" {}", parts.join("; ")) +} diff --git a/crates/sqlbuild-core/src/progress/main/format_expectation_name.rs b/crates/sqlbuild-core/src/progress/main/format_expectation_name.rs new file mode 100644 index 0000000000..335d42f6e6 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/format_expectation_name.rs @@ -0,0 +1,12 @@ +//! The user-facing name of one SQL expectation row. + +use crate::progress::constants::ASSERTION_PREFIX; + +/// Assertions keep their name; compared models read as `expected `. +#[must_use] +pub fn format_expectation_name(model_name: &str) -> String { + if model_name.starts_with(ASSERTION_PREFIX) { + return model_name.to_owned(); + } + format!("expected {model_name}") +} diff --git a/crates/sqlbuild-core/src/progress/main/format_inspection_record.rs b/crates/sqlbuild-core/src/progress/main/format_inspection_record.rs new file mode 100644 index 0000000000..8edc8c928a --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/format_inspection_record.rs @@ -0,0 +1,34 @@ +//! One verbose warehouse inspection line. + +use crate::progress::constants::SQL_PREVIEW_LENGTH; +use crate::progress::models::InspectionRecord; + +fn collapse_whitespace(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") +} + +fn row_count_text(row_count: Option) -> String { + match row_count { + None => "? rows".to_owned(), + Some(1) => "1 row".to_owned(), + Some(count) => format!("{count} rows"), + } +} + +/// ` inspect 0.12s 3 rows SELECT ...`, with the SQL on one line and cut to the preview length. +#[must_use] +pub fn format_inspection_record(record: &InspectionRecord) -> String { + let mut text: String = collapse_whitespace(&record.sql); + if text.chars().count() > SQL_PREVIEW_LENGTH { + let kept: String = text.chars().take(SQL_PREVIEW_LENGTH - 3).collect(); + text = format!("{kept}..."); + } + let outcome: String = record.error.as_ref().map_or_else( + || row_count_text(record.row_count), + |error| format!("failed: {}", collapse_whitespace(error)), + ); + format!( + " inspect {:6.2}s {outcome} {text}", + record.elapsed_seconds + ) +} diff --git a/crates/sqlbuild-core/src/progress/main/is_planning_completion_message.rs b/crates/sqlbuild-core/src/progress/main/is_planning_completion_message.rs new file mode 100644 index 0000000000..53b58bcc86 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/is_planning_completion_message.rs @@ -0,0 +1,11 @@ +//! Whether a planning progress message completes its phase. + +use crate::progress::constants::PLANNING_COMPLETION_PREFIXES; + +/// Completion messages start with a past-tense verb such as `Planned ` or `Compiled `. +#[must_use] +pub fn is_planning_completion_message(message: &str) -> bool { + PLANNING_COMPLETION_PREFIXES + .iter() + .any(|prefix| message.starts_with(prefix)) +} diff --git a/crates/sqlbuild-core/src/progress/main/mod.rs b/crates/sqlbuild-core/src/progress/main/mod.rs new file mode 100644 index 0000000000..fdbaea3b79 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/mod.rs @@ -0,0 +1,12 @@ +pub mod claim_progress_terminal; +pub mod close_progress_projection; +pub mod connection_phase_line; +pub mod connection_start_message; +pub mod consume_progress_event; +pub mod execution_header_text; +pub mod format_execution_header; +pub mod format_expectation_detail; +pub mod format_expectation_name; +pub mod format_inspection_record; +pub mod is_planning_completion_message; +pub mod progress_terminal_duration; diff --git a/crates/sqlbuild-core/src/progress/main/progress_terminal_duration.rs b/crates/sqlbuild-core/src/progress/main/progress_terminal_duration.rs new file mode 100644 index 0000000000..43495c62e5 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/progress_terminal_duration.rs @@ -0,0 +1,15 @@ +//! The duration of the terminal a result would claim, without claiming it. + +use crate::progress::_helpers::resources::resolve_resource_terminal; +use crate::progress::models::ProgressProjection; +use crate::terminal_events::models::TerminalRequest; + +/// The duration of the terminal `request` selects, leaving its presentation unclaimed. +#[must_use] +pub fn progress_terminal_duration( + projection: &ProgressProjection, + request: &TerminalRequest, +) -> Option { + let (_, candidates, target) = resolve_resource_terminal(projection, request)?; + candidates[target].payload.duration_ms +} diff --git a/crates/sqlbuild-core/src/progress/mod.rs b/crates/sqlbuild-core/src/progress/mod.rs new file mode 100644 index 0000000000..1644773e62 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/progress/models.rs b/crates/sqlbuild-core/src/progress/models.rs new file mode 100644 index 0000000000..7ec2be6f8f --- /dev/null +++ b/crates/sqlbuild-core/src/progress/models.rs @@ -0,0 +1,144 @@ +//! The lifecycle facts progress projection reads, and the invocation projection state. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use crate::progress::types::DifferenceSample; + +/// The payload fields of one lifecycle event that progress projection reads. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ProgressPayload { + /// `resource_name`, when it is a string. + pub resource_name: Option, + /// `resource_kind`, when it is a string. + pub resource_kind: Option, + /// `duration_ms`, when it is a number. + pub duration_ms: Option, + /// `attempt_number`, when it is an integral number. + pub attempt_number: Option, + /// `operation_name`, when it is a string. + pub operation_name: Option, + /// `metadata.attempt_number`, when it is an `int` other than `bool`. + pub metadata_attempt_number: Option, + /// `metadata.item_count`, when it is an `int` other than `bool`. + pub metadata_item_count: Option, + /// `metadata.unknown_count`, when it is an `int` other than `bool`. + pub metadata_unknown_count: Option, + /// `error_type`, when it is a string. + pub error_type: Option, + /// `failed_attempt_number`, when it is an `int` other than `bool`. + pub failed_attempt_number: Option, + /// `next_attempt_number`, when it is an `int` other than `bool`. + pub next_attempt_number: Option, + /// `delay_ms`, when it is an `int` other than `bool`. + pub delay_ms: Option, + /// `query_id`, when it is a string. + pub query_id: Option, + /// `phase`, when it is a string. + pub phase: Option, + /// The display text of `intent`, `execute` when absent. + pub intent: String, + /// The display text of `statement_kind`, `UNKNOWN` when absent. + pub statement_kind: String, +} + +/// The correlations and payload of one lifecycle event that progress projection reads. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ProgressEvent { + pub event_id: String, + pub event_type: String, + pub resource_id: Option, + pub resource_attempt_id: Option, + pub operation_id: Option, + pub statement_id: Option, + pub payload: ProgressPayload, +} + +/// How the progress stream renders. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ProgressSettings { + pub use_color: bool, + pub debug: bool, + /// Whether the stream is a terminal, where live owners draw transient rows instead. + pub is_tty: bool, +} + +/// The invocation-scoped projection of canonical lifecycle facts onto persistent progress lines. +#[derive(Clone, Debug, Default)] +pub struct ProgressProjection { + pub(crate) settings: ProgressSettings, + pub(crate) seen_event_ids: HashSet, + pub(crate) resource_ordinals: HashMap, + pub(crate) resource_total: i64, + /// Attempt identifiers in first-seen order, which `close` renders in. + pub(crate) attempt_order: Vec, + pub(crate) attempts_by_id: HashMap, + pub(crate) pending_terminal_ids_by_resource: HashMap>, + pub(crate) resource_ids_by_name: HashMap>, + pub(crate) rendered_terminal_ids: HashSet, + pub(crate) operation_starts: HashMap, + pub(crate) quiet_operation_ids: HashSet, + pub(crate) enriched_resource_names: HashSet, + pub(crate) resource_names: HashMap, + pub(crate) statement_starts: HashMap, + pub(crate) statement_query_ids: HashMap, +} + +impl ProgressProjection { + /// An empty projection rendering with `settings`. + #[must_use] + pub fn new(settings: ProgressSettings) -> Self { + Self { + settings, + ..Self::default() + } + } + + /// Add display order metadata without changing canonical facts. + pub fn configure_resources(&mut self, ordinals: Vec<(String, i64)>, total: i64) { + for (name, ordinal) in ordinals { + self.enriched_resource_names.insert(name.clone()); + self.resource_ordinals.insert(name, ordinal); + } + self.resource_total = self.resource_total.max(total); + } + + /// Declare that a result callback will enrich this resource's terminal. + pub fn expect_resource_enrichment(&mut self, resource_name: &str) { + self.enriched_resource_names + .insert(resource_name.to_owned()); + } + + /// Whether a visible operation with this name currently owns progress. + #[must_use] + pub fn is_operation_active(&self, operation_name: &str) -> bool { + self.operation_starts + .values() + .any(|event| event.payload.operation_name.as_deref() == Some(operation_name)) + } +} + +/// The fields of one SQL test chain step that its expectation row reports. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ExpectationStep { + pub model_name: String, + /// The outcome value: `pass`, `fail` or `error`. + pub outcome: String, + pub actual_row_count: i64, + pub expected_row_count: i64, + pub mismatched_row_count: i64, + pub unexpected_row_count: Option, + pub missing_row_count: Option, + pub unexpected_samples: Vec, + pub missing_samples: Vec, + /// Differing columns as name, actual and expected value. + pub column_differences: Vec<(String, String, String)>, +} + +/// One warehouse inspection read reported in verbose planning output. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct InspectionRecord { + pub sql: String, + pub elapsed_seconds: f64, + pub row_count: Option, + pub error: Option, +} diff --git a/crates/sqlbuild-core/src/progress/tests.rs b/crates/sqlbuild-core/src/progress/tests.rs new file mode 100644 index 0000000000..992daf34c1 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_progress.rs"] +mod progress; +#[path = "tests/test_progress_text.rs"] +mod progress_text; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/progress/tests/helpers.rs b/crates/sqlbuild-core/src/progress/tests/helpers.rs new file mode 100644 index 0000000000..a654c48407 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/tests/helpers.rs @@ -0,0 +1,86 @@ +use crate::progress::main::claim_progress_terminal::claim_progress_terminal; +use crate::progress::main::close_progress_projection::close_progress_projection; +use crate::progress::main::consume_progress_event::consume_progress_event; +use crate::progress::models::{ + ProgressEvent, ProgressPayload, ProgressProjection, ProgressSettings, +}; +use crate::progress::tests::test_types::ProjectionTestCase; +use crate::terminal_events::models::TerminalRequest; + +/// One `orders` resource attempt event. +pub(super) fn attempt_event( + event_id: &str, + event_type: &str, + (attempt_id, duration_ms): (&str, Option), +) -> ProgressEvent { + ProgressEvent { + event_id: event_id.to_owned(), + event_type: event_type.to_owned(), + resource_id: Some("model:orders".to_owned()), + resource_attempt_id: Some(attempt_id.to_owned()), + payload: ProgressPayload { + resource_name: Some("orders".to_owned()), + resource_kind: Some("table".to_owned()), + duration_ms, + ..ProgressPayload::default() + }, + ..ProgressEvent::default() + } +} + +/// One operation event, optionally inside the `orders` attempt. +pub(super) fn operation_event( + event_id: &str, + event_type: &str, + (operation_name, attempt_id): (&str, Option<&str>), +) -> ProgressEvent { + ProgressEvent { + event_id: event_id.to_owned(), + event_type: event_type.to_owned(), + resource_id: attempt_id.map(|_| "model:orders".to_owned()), + resource_attempt_id: attempt_id.map(str::to_owned), + operation_id: Some("op1".to_owned()), + payload: ProgressPayload { + operation_name: Some(operation_name.to_owned()), + duration_ms: Some(1500.0), + ..ProgressPayload::default() + }, + ..ProgressEvent::default() + } +} + +/// Every line a case writes, then the durations its claims return. +pub(super) fn project_case(test_case: &ProjectionTestCase) -> (Vec, Vec>) { + let mut projection: ProgressProjection = ProgressProjection::new(test_case.settings); + for name in &test_case.enriched { + projection.expect_resource_enrichment(name); + } + let mut lines: Vec = Vec::new(); + for event in &test_case.events { + let (consumed, event_lines) = consume_progress_event(projection, event); + projection = consumed; + lines.extend(event_lines); + } + let mut durations: Vec> = Vec::new(); + for request in &test_case.claims { + let (claimed, claim_lines, duration) = claim_progress_terminal(projection, request); + projection = claimed; + lines.extend(claim_lines); + durations.push(duration); + } + lines.extend(close_progress_projection(projection).1); + (lines, durations) +} + +/// A claim for `orders` by name only. +pub(super) fn orders_claim() -> TerminalRequest { + TerminalRequest { + resource_name: "orders".to_owned(), + ..TerminalRequest::default() + } +} + +/// Settings of a plain, non-terminal, non-debug stream. +pub(super) fn plain() -> ProgressSettings { + ProgressSettings::default() +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress.rs b/crates/sqlbuild-core/src/progress/tests/test_progress.rs new file mode 100644 index 0000000000..a3ff9d8de3 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/tests/test_progress.rs @@ -0,0 +1,213 @@ +use crate::progress::models::{ProgressEvent, ProgressPayload, ProgressSettings}; +use crate::progress::tests::helpers::{ + attempt_event, operation_event, orders_claim, plain, project_case, +}; +use crate::progress::tests::test_types::ProjectionTestCase; + +#[test] +fn given_lifecycle_facts_when_projecting_then_progress_lines_match() { + let test_cases = [ + ProjectionTestCase { + description: "an unenriched attempt renders its start and terminal once", + settings: plain(), + enriched: vec![], + events: vec![ + attempt_event("e1", "resource_attempt_started", ("a1", None)), + attempt_event("e2", "resource_attempt_completed", ("a1", Some(1234.0))), + attempt_event("e2", "resource_attempt_completed", ("a1", Some(1234.0))), + ], + claims: vec![], + expected_lines: vec![" table orders START", " table orders OK 1.23s"], + expected_durations: vec![], + }, + ProjectionTestCase { + description: "an enriched terminal is claimed and earlier attempts render first", + settings: plain(), + enriched: vec!["orders"], + events: vec![ + attempt_event("e1", "resource_attempt_failed", ("a1", Some(10.0))), + attempt_event("e2", "resource_attempt_completed", ("a2", Some(20.0))), + ], + claims: vec![orders_claim(), orders_claim()], + expected_lines: vec![" table orders FAIL 0.01s"], + expected_durations: vec![Some(20.0), None], + }, + ProjectionTestCase { + description: "an unclaimed enriched terminal renders on close", + settings: plain(), + enriched: vec!["orders"], + events: vec![attempt_event( + "e1", + "resource_attempt_skipped", + ("a1", None), + )], + claims: vec![], + expected_lines: vec![" table orders SKIP"], + expected_durations: vec![], + }, + ProjectionTestCase { + description: "a terminal stream leaves starts to live rows", + settings: ProgressSettings { + is_tty: true, + ..plain() + }, + enriched: vec![], + events: vec![ + attempt_event("e1", "resource_attempt_started", ("a1", None)), + operation_event("o1", "operation_started", ("project_compile", None)), + operation_event("o2", "operation_completed", ("project_compile", None)), + ], + claims: vec![], + expected_lines: vec![], + expected_durations: vec![], + }, + ProjectionTestCase { + description: "a top-level operation renders its start and terminal", + settings: plain(), + enriched: vec![], + events: vec![ + operation_event("o1", "operation_started", ("project_compile", None)), + operation_event("o2", "operation_failed", ("project_compile", None)), + ], + claims: vec![], + expected_lines: vec!["Project compile START", "Project compile FAIL (1.50s)"], + expected_durations: vec![], + }, + ProjectionTestCase { + description: "a quiet resource operation renders only its failure", + settings: plain(), + enriched: vec!["orders"], + events: vec![ + attempt_event("e1", "resource_attempt_started", ("a1", None)), + operation_event("o1", "operation_started", ("staging_creation", Some("a1"))), + operation_event("o2", "operation_failed", ("staging_creation", Some("a1"))), + ], + claims: vec![], + expected_lines: vec![ + " table orders START", + " Create staging relation FAIL (1.50s)", + ], + expected_durations: vec![], + }, + ProjectionTestCase { + description: "freshness metadata and retries render their details", + settings: plain(), + enriched: vec![], + events: vec![ + operation_event( + "o1", + "operation_started", + ("source_freshness_metadata_observation", None), + ), + ProgressEvent { + payload: ProgressPayload { + metadata_item_count: Some(1), + metadata_unknown_count: Some(1), + ..operation_event( + "o2", + "operation_completed", + ("source_freshness_metadata_observation", None), + ) + .payload + }, + ..operation_event( + "o2", + "operation_completed", + ("source_freshness_metadata_observation", None), + ) + }, + ProgressEvent { + payload: ProgressPayload { + failed_attempt_number: Some(1), + next_attempt_number: Some(2), + delay_ms: Some(250.0), + ..ProgressPayload::default() + }, + ..attempt_event("r1", "retry_scheduled", ("a1", None)) + }, + ], + claims: vec![], + expected_lines: vec![ + "Freshness metadata START", + "Freshness metadata OK (1 source, 1 unknown) (1.50s)", + " retry 1->2 in 0.25s", + ], + expected_durations: vec![], + }, + ProjectionTestCase { + description: "debug statements render start, query id, heartbeat and failure", + settings: ProgressSettings { + debug: true, + ..plain() + }, + enriched: vec![], + events: vec![ + attempt_event("e1", "resource_attempt_started", ("a1", None)), + ProgressEvent { + statement_id: Some("s1".to_owned()), + payload: ProgressPayload { + intent: "execute".to_owned(), + statement_kind: "CREATE".to_owned(), + ..ProgressPayload::default() + }, + ..attempt_event("s-start", "statement_started", ("a1", None)) + }, + ProgressEvent { + statement_id: Some("s1".to_owned()), + payload: ProgressPayload { + query_id: Some("q1".to_owned()), + intent: "execute".to_owned(), + statement_kind: "CREATE".to_owned(), + ..ProgressPayload::default() + }, + ..attempt_event("s-sub", "statement_submitted", ("a1", None)) + }, + ProgressEvent { + statement_id: Some("s1".to_owned()), + payload: ProgressPayload { + duration_ms: Some(31000.0), + intent: "execute".to_owned(), + statement_kind: "CREATE".to_owned(), + ..ProgressPayload::default() + }, + ..attempt_event("s-hb", "statement_heartbeat", ("a1", None)) + }, + ProgressEvent { + statement_id: Some("s1".to_owned()), + payload: ProgressPayload { + intent: "execute".to_owned(), + statement_kind: "CREATE".to_owned(), + ..ProgressPayload::default() + }, + ..attempt_event("s-fail", "statement_failed", ("a1", None)) + }, + ], + claims: vec![], + expected_lines: vec![ + " table orders START", + " statement model=orders phase=execute kind=CREATE START", + " statement model=orders phase=execute kind=CREATE query_id=q1", + " statement model=orders phase=execute kind=CREATE RUNNING 31.0s query_id=q1", + " statement model=orders phase=execute kind=CREATE FAIL query_id=q1", + ], + expected_durations: vec![], + }, + ]; + + for test_case in test_cases { + let (lines, durations) = project_case(&test_case); + assert_eq!( + (lines, durations), + ( + test_case + .expected_lines + .iter() + .map(|line| (*line).to_owned()) + .collect::>(), + test_case.expected_durations + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs new file mode 100644 index 0000000000..7660f98bcd --- /dev/null +++ b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs @@ -0,0 +1,200 @@ +use crate::cli_text::models::CliStyle; +use crate::progress::main::connection_phase_line::connection_phase_line; +use crate::progress::main::connection_start_message::connection_start_message; +use crate::progress::main::execution_header_text::execution_header_text; +use crate::progress::main::format_expectation_detail::format_expectation_detail; +use crate::progress::main::format_expectation_name::format_expectation_name; +use crate::progress::main::format_inspection_record::format_inspection_record; +use crate::progress::main::is_planning_completion_message::is_planning_completion_message; +use crate::progress::models::{ExpectationStep, InspectionRecord}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; +use crate::progress::tests::test_types::{ + CompletionMessageTestCase, ExpectationDetailTestCase, InspectionRecordTestCase, + ProgressLineTestCase, +}; + +#[test] +fn given_planning_message_when_classifying_then_completions_are_recognized() { + let test_cases = [ + CompletionMessageTestCase { + description: "an applied clone plan completes", + message: "Applied clone plan. (0.10s)", + expected_is_completion: true, + }, + CompletionMessageTestCase { + description: "a finalized dbt run completes", + message: "Finalized dbt run.", + expected_is_completion: true, + }, + CompletionMessageTestCase { + description: "evaluated rules complete", + message: "Evaluated rules. (built-in 0.00s, custom 0.00s)", + expected_is_completion: true, + }, + CompletionMessageTestCase { + description: "an ongoing phase does not complete", + message: "Compiling project...", + expected_is_completion: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + is_planning_completion_message(test_case.message), + test_case.expected_is_completion, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_connection_and_header_when_formatting_then_lines_match() { + let test_cases = [ + ProgressLineTestCase { + description: "a single connection omits its count", + line: connection_start_message("duckdb", 1), + expected_line: "Connecting to duckdb...", + }, + ProgressLineTestCase { + description: "several connections show their count", + line: connection_start_message("databricks", 8), + expected_line: "Connecting to databricks (8 connections)...", + }, + ProgressLineTestCase { + description: "a connected warehouse reports its elapsed time", + line: connection_phase_line(PLAIN, true, ("duckdb", 0.034)), + expected_line: "\u{2713} Warehouse connected duckdb (0.03s)", + }, + ProgressLineTestCase { + description: "a failed connection reports the time it took", + line: connection_phase_line(PLAIN, false, ("duckdb", 18.424)), + expected_line: "\u{2717} Warehouse connection failed duckdb (after 18.42s)", + }, + ProgressLineTestCase { + description: "a header with a target lists it before concurrency", + line: execution_header_text(PLAIN, ("sqb build", Some("dev"), 4)), + expected_line: "Execution sqb build (target: dev, concurrency: 4)\n\n", + }, + ProgressLineTestCase { + description: "a header without a target lists only concurrency", + line: execution_header_text(PLAIN, ("sqb test", None, 1)), + expected_line: "Execution sqb test (concurrency: 1)\n\n", + }, + ProgressLineTestCase { + description: "a compared model reads as expected", + line: format_expectation_name("orders"), + expected_line: "expected orders", + }, + ProgressLineTestCase { + description: "an assertion keeps its name", + line: format_expectation_name("assertion totals_are_positive"), + expected_line: "assertion totals_are_positive", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.line, test_case.expected_line, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_expectation_step_when_formatting_detail_then_detail_matches() { + let failing: ExpectationStep = ExpectationStep { + model_name: "orders".to_owned(), + outcome: "fail".to_owned(), + actual_row_count: 2, + expected_row_count: 3, + ..ExpectationStep::default() + }; + let test_cases = [ + ExpectationDetailTestCase { + description: "a passing step has no detail", + step: ExpectationStep { + outcome: "pass".to_owned(), + ..failing.clone() + }, + expected_detail: "", + }, + ExpectationDetailTestCase { + description: "an assertion reports its row count", + step: ExpectationStep { + model_name: "assertion totals".to_owned(), + actual_row_count: 1, + ..failing.clone() + }, + expected_detail: " 1 row", + }, + ExpectationDetailTestCase { + description: "a step without set differences reports mismatched rows", + step: ExpectationStep { + mismatched_row_count: 4, + ..failing.clone() + }, + expected_detail: " 4 mismatched", + }, + ExpectationDetailTestCase { + description: "set differences list counts, samples and columns", + step: ExpectationStep { + unexpected_row_count: Some(1), + missing_samples: vec![vec![("id".to_owned(), "7".to_owned())]], + column_differences: vec![("total".to_owned(), "1".to_owned(), "2".to_owned())], + ..failing.clone() + }, + expected_detail: " unexpected=1; missing=None; row counts actual=2 expected=3; \ + missing sample 1: id=7; differing column total: actual=1, expected=2", + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_expectation_detail(&test_case.step), + test_case.expected_detail, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_inspection_record_when_formatting_then_sql_is_collapsed_and_cut() { + let test_cases = [ + InspectionRecordTestCase { + description: "a read reports its row count on one line", + record: InspectionRecord { + sql: "SELECT 1\n FROM orders".to_owned(), + elapsed_seconds: 0.125, + row_count: Some(1), + error: None, + }, + expected_line: " inspect 0.12s 1 row SELECT 1 FROM orders".to_owned(), + }, + InspectionRecordTestCase { + description: "a failed read reports its error and long SQL is cut", + record: InspectionRecord { + sql: "x".repeat(200), + elapsed_seconds: 12.5, + row_count: None, + error: Some("no\nsuch table".to_owned()), + }, + expected_line: format!( + " inspect 12.50s failed: no such table {}...", + "x".repeat(157) + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_inspection_record(&test_case.record), + test_case.expected_line, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_types.rs b/crates/sqlbuild-core/src/progress/tests/test_types.rs new file mode 100644 index 0000000000..d65c94e451 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/tests/test_types.rs @@ -0,0 +1,36 @@ +use crate::progress::models::{ProgressEvent, ProgressSettings}; +use crate::terminal_events::models::TerminalRequest; + +pub(super) struct ProjectionTestCase { + pub(super) description: &'static str, + pub(super) settings: ProgressSettings, + pub(super) enriched: Vec<&'static str>, + pub(super) events: Vec, + pub(super) claims: Vec, + pub(super) expected_lines: Vec<&'static str>, + pub(super) expected_durations: Vec>, +} + +pub(super) struct CompletionMessageTestCase { + pub(super) description: &'static str, + pub(super) message: &'static str, + pub(super) expected_is_completion: bool, +} + +pub(super) struct ExpectationDetailTestCase { + pub(super) description: &'static str, + pub(super) step: crate::progress::models::ExpectationStep, + pub(super) expected_detail: &'static str, +} + +pub(super) struct InspectionRecordTestCase { + pub(super) description: &'static str, + pub(super) record: crate::progress::models::InspectionRecord, + pub(super) expected_line: String, +} + +pub(super) struct ProgressLineTestCase { + pub(super) description: &'static str, + pub(super) line: String, + pub(super) expected_line: &'static str, +} diff --git a/crates/sqlbuild-core/src/progress/types.rs b/crates/sqlbuild-core/src/progress/types.rs new file mode 100644 index 0000000000..fad8ca1b1f --- /dev/null +++ b/crates/sqlbuild-core/src/progress/types.rs @@ -0,0 +1,4 @@ +//! Type aliases of progress text inputs. + +/// One redacted representative row of an expected-output difference. +pub type DifferenceSample = Vec<(String, String)>; diff --git a/crates/sqlbuild-cursor/Cargo.toml b/crates/sqlbuild-cursor/Cargo.toml new file mode 100644 index 0000000000..472f1afaf6 --- /dev/null +++ b/crates/sqlbuild-cursor/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "sqlbuild-cursor" +version.workspace = true +description = "Microbatch cursor values, grains, half-open intervals and durations, as Python's datetime computes them." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +sqlbuild-core.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-cursor/src/algebra/_helpers/arithmetic.rs b/crates/sqlbuild-cursor/src/algebra/_helpers/arithmetic.rs new file mode 100644 index 0000000000..3ee28625fb --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/_helpers/arithmetic.rs @@ -0,0 +1,201 @@ +//! Flooring and stepping one cursor value by its grain. + +use crate::algebra::_helpers::civil::{ + add_days, add_micros, checked_date, timedelta_days, timedelta_micros, +}; +use crate::algebra::_helpers::failures::{algebra, overflow}; +use crate::algebra::constants::{ + INTEGER_CURSOR_OVERFLOW, MICROS_PER_DAY, MICROS_PER_SECOND, MONTHS_PER_YEAR, NO_GRAIN_STEP, + SECONDS_PER_HOUR, SECONDS_PER_MINUTE, TEMPORAL_VALUES_NEED_GRAIN, +}; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{CivilDate, Grain, LocalDateTime, Scalar, ScalarValue, Timestamp}; + +/// The fixed length of one grain in microseconds; months and years vary. +pub(crate) fn fixed_step(grain: Grain) -> Option { + match grain { + Grain::Second => Some(MICROS_PER_SECOND), + Grain::Minute => Some(SECONDS_PER_MINUTE * MICROS_PER_SECOND), + Grain::Hour => Some(SECONDS_PER_HOUR * MICROS_PER_SECOND), + Grain::Day => Some(MICROS_PER_DAY), + Grain::Month | Grain::Year => None, + } +} + +/// A new value with no input handle. +pub(crate) fn fresh(value: ScalarValue) -> Scalar { + Scalar { + value, + handle: None, + } +} + +/// A new timestamp in `source`'s zone; `datetime + timedelta` resets `fold`, `replace` keeps it. +pub(crate) fn retimed(source: &Timestamp, local: LocalDateTime, fold: bool) -> Scalar { + fresh(ScalarValue::Timestamp(Timestamp { + local, + fold, + offset: source.offset, + template: source.template, + source_text: None, + })) +} + +pub(crate) fn integer_sum(value: i128, delta: i128) -> Result { + value + .checked_add(delta) + .ok_or_else(|| overflow(INTEGER_CURSOR_OVERFLOW)) +} + +fn floor_local(local: &LocalDateTime, grain: Grain) -> LocalDateTime { + let mut floored: LocalDateTime = *local; + floored.microsecond = 0; + if grain >= Grain::Minute { + floored.second = 0; + } + if grain >= Grain::Hour { + floored.minute = 0; + } + if grain >= Grain::Day { + floored.hour = 0; + } + if grain >= Grain::Month { + floored.date.day = 1; + } + if grain == Grain::Year { + floored.date.month = 1; + } + floored +} + +/// The aligned value at or before `value`; without a grain a timestamp is only copied. +pub(crate) fn floor_scalar(value: &Scalar, grain: Option) -> Scalar { + match (&value.value, grain) { + (ScalarValue::Integer(_), _) => value.clone(), + (ScalarValue::Date(date), Some(grain)) if grain >= Grain::Month => { + let month: u32 = if grain == Grain::Year { 1 } else { date.month }; + fresh(ScalarValue::Date(CivilDate { + year: date.year, + month, + day: 1, + })) + } + (ScalarValue::Date(_), _) => value.clone(), + (ScalarValue::Timestamp(timestamp), None) => { + retimed(timestamp, timestamp.local, timestamp.fold) + } + (ScalarValue::Timestamp(timestamp), Some(grain)) => retimed( + timestamp, + floor_local(×tamp.local, grain), + timestamp.fold, + ), + } +} + +/// `value.replace(year=..., month=..., day=1)` `steps` months after the value's month. +fn advance_months(date: CivilDate, steps: i64) -> Result { + let index: i64 = date.year * MONTHS_PER_YEAR + i64::from(date.month) - 1 + steps; + let month: u32 = (index.rem_euclid(MONTHS_PER_YEAR) + 1) as u32; + checked_date(index.div_euclid(MONTHS_PER_YEAR), month, 1) +} + +/// The value one grain after an aligned `value`; without a grain dates step a day. +pub(crate) fn advance_scalar(value: &Scalar, grain: Option) -> Result { + match &value.value { + ScalarValue::Integer(number) => Ok(fresh(ScalarValue::Integer(integer_sum(*number, 1)?))), + ScalarValue::Date(date) => { + let advanced: CivilDate = match grain { + Some(Grain::Month) => advance_months(*date, 1)?, + Some(Grain::Year) => checked_date(date.year + 1, 1, 1)?, + _ => add_days(*date, 1)?, + }; + Ok(fresh(ScalarValue::Date(advanced))) + } + ScalarValue::Timestamp(timestamp) => { + let local: &LocalDateTime = ×tamp.local; + let Some(grain) = grain else { + return Err(CursorError::Type(NO_GRAIN_STEP.to_owned())); + }; + let Some(step) = fixed_step(grain) else { + let date: CivilDate = match grain { + Grain::Month => advance_months(local.date, 1)?, + _ => checked_date(local.date.year + 1, 1, 1)?, + }; + let advanced = LocalDateTime { date, ..*local }; + return Ok(retimed(timestamp, advanced, timestamp.fold)); + }; + Ok(retimed(timestamp, add_micros(local, step)?, false)) + } + } +} + +/// The value `steps` grains after an aligned `value`, in constant time. +pub(crate) fn advance_scalar_by( + value: &Scalar, + grain: Option, + steps: i64, +) -> Result { + if let ScalarValue::Integer(number) = &value.value { + return Ok(fresh(ScalarValue::Integer(integer_sum( + *number, + i128::from(steps), + )?))); + } + let Some(grain) = grain else { + return Err(algebra(TEMPORAL_VALUES_NEED_GRAIN)); + }; + match &value.value { + ScalarValue::Date(date) => Ok(fresh(ScalarValue::Date(match grain { + Grain::Month => advance_months(*date, steps)?, + Grain::Year => checked_date(date.year + steps, date.month, date.day)?, + _ => add_days( + *date, + timedelta_days(timedelta_micros(i128::from(steps) * MICROS_PER_DAY)?), + )?, + }))), + ScalarValue::Timestamp(timestamp) => { + let local: &LocalDateTime = ×tamp.local; + let Some(step) = fixed_step(grain) else { + let date: CivilDate = match grain { + Grain::Month => advance_months(local.date, steps)?, + _ => checked_date(local.date.year + steps, local.date.month, local.date.day)?, + }; + let advanced = LocalDateTime { date, ..*local }; + return Ok(retimed(timestamp, advanced, timestamp.fold)); + }; + let delta: i128 = timedelta_micros(step * i128::from(steps))?; + Ok(retimed(timestamp, add_micros(local, delta)?, false)) + } + ScalarValue::Integer(_) => Ok(value.clone()), + } +} + +/// An inclusive or exclusive bound moved one compatible unit in `direction`. +pub(crate) fn shift_bound( + value: &Scalar, + grain: Option, + direction: i64, + missing_grain: &str, +) -> Result { + if let ScalarValue::Integer(number) = &value.value { + return Ok(fresh(ScalarValue::Integer(integer_sum( + *number, + i128::from(direction), + )?))); + } + let Some(grain) = grain else { + return Err(algebra(missing_grain)); + }; + match &value.value { + ScalarValue::Date(date) => Ok(fresh(ScalarValue::Date(add_days( + *date, + i128::from(direction), + )?))), + ScalarValue::Timestamp(timestamp) => { + let step: i128 = fixed_step(grain).unwrap_or(MICROS_PER_DAY); + let local: LocalDateTime = add_micros(×tamp.local, i128::from(direction) * step)?; + Ok(retimed(timestamp, local, false)) + } + ScalarValue::Integer(_) => Ok(value.clone()), + } +} diff --git a/crates/sqlbuild-cursor/src/algebra/_helpers/civil.rs b/crates/sqlbuild-cursor/src/algebra/_helpers/civil.rs new file mode 100644 index 0000000000..bc78514c06 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/_helpers/civil.rs @@ -0,0 +1,166 @@ +//! Proleptic Gregorian dates and microsecond wall times within `datetime`'s year range. + +use crate::algebra::_helpers::failures::overflow; +use crate::algebra::constants::{ + C_INT_OVERFLOW, CENTURY_YEARS, DATE_OUT_OF_RANGE, DAY_OUT_OF_RANGE, DAYS_IN_MONTH, FEBRUARY, + FIRST_NEXT_YEAR_INDEX, LEAP_CYCLE_YEARS, LEAP_YEARS, MARCH, MAX_C_INT, MAX_TIMEDELTA_DAYS, + MAX_YEAR, MICROS_PER_DAY, MICROS_PER_SECOND, MIN_C_INT, MIN_YEAR, SECONDS_PER_HOUR, + SECONDS_PER_MINUTE, +}; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{CivilDate, LocalDateTime}; + +/// Days from 0000-03-01 to 1970-01-01 in Howard Hinnant's civil-day algorithm. +const UNIX_EPOCH_SHIFT: i64 = 719_468; +/// `date(1970, 1, 1).toordinal()`. +const UNIX_EPOCH_ORDINAL: i64 = 719_163; +const DAYS_PER_ERA: i64 = 146_097; +const DAYS_PER_YEAR: i64 = 365; +const MARCH_BASED_MONTH_DAYS: i64 = 153; +const MARCH_OFFSET: i64 = 3; +const MONTHS_FROM_MARCH: i64 = 9; +const MARCH_DAY_ROUNDING: i64 = 2; +const MONTH_LENGTH_DIVISOR: i64 = 5; +const ERA_DAY_LIMIT: i64 = 146_096; +const FOUR_YEAR_DAYS: i64 = 1_460; +const CENTURY_DAYS: i64 = 36_524; + +/// `calendar.isleap` for any year. +pub(crate) fn is_leap(year: i64) -> bool { + year.rem_euclid(LEAP_YEARS) == 0 + && (year.rem_euclid(CENTURY_YEARS) != 0 || year.rem_euclid(LEAP_CYCLE_YEARS) == 0) +} + +/// `calendar.monthrange(year, month)[1]` for a month in `1..=12`. +pub(crate) fn days_in_month(year: i64, month: u32) -> u32 { + let base: u32 = DAYS_IN_MONTH[(month - 1) as usize]; + base + u32::from(month == FEBRUARY && is_leap(year)) +} + +/// `date.replace` validation: the year first, then the day of the month. +pub(crate) fn checked_date(year: i64, month: u32, day: u32) -> Result { + if !(MIN_YEAR..=MAX_YEAR).contains(&year) { + return Err(CursorError::Value(format!("year {year} is out of range"))); + } + if day < 1 || day > days_in_month(year, month) { + return Err(CursorError::Value(DAY_OUT_OF_RANGE.to_owned())); + } + Ok(CivilDate { year, month, day }) +} + +/// `date.toordinal()`. +pub(crate) fn ordinal(date: CivilDate) -> i64 { + let year: i64 = date.year - i64::from(date.month <= FEBRUARY); + let era: i64 = year.div_euclid(LEAP_CYCLE_YEARS); + let year_of_era: i64 = year - era * LEAP_CYCLE_YEARS; + let month: i64 = i64::from(date.month); + let shifted_month: i64 = if month >= i64::from(MARCH) { + month - MARCH_OFFSET + } else { + month + MONTHS_FROM_MARCH + }; + let day_of_year: i64 = (MARCH_BASED_MONTH_DAYS * shifted_month + MARCH_DAY_ROUNDING) + / MONTH_LENGTH_DIVISOR + + i64::from(date.day) + - 1; + let day_of_era: i64 = year_of_era * DAYS_PER_YEAR + year_of_era / LEAP_YEARS + - year_of_era / CENTURY_YEARS + + day_of_year; + era * DAYS_PER_ERA + day_of_era - UNIX_EPOCH_SHIFT + UNIX_EPOCH_ORDINAL +} + +/// `date.fromordinal(value)`, failing as date arithmetic does outside the year range. +pub(crate) fn from_ordinal(value: i128) -> Result { + let maximum: i128 = i128::from(ordinal(CivilDate { + year: MAX_YEAR, + month: 12, + day: 31, + })); + if !(1..=maximum).contains(&value) { + return Err(overflow(DATE_OUT_OF_RANGE)); + } + let days: i64 = i64::try_from(value).map_err(|_| overflow(DATE_OUT_OF_RANGE))? + - UNIX_EPOCH_ORDINAL + + UNIX_EPOCH_SHIFT; + let era: i64 = days.div_euclid(DAYS_PER_ERA); + let day_of_era: i64 = days - era * DAYS_PER_ERA; + let year_of_era: i64 = (day_of_era - day_of_era / FOUR_YEAR_DAYS + day_of_era / CENTURY_DAYS + - day_of_era / ERA_DAY_LIMIT) + / DAYS_PER_YEAR; + let day_of_year: i64 = day_of_era + - (DAYS_PER_YEAR * year_of_era + year_of_era / LEAP_YEARS - year_of_era / CENTURY_YEARS); + let shifted_month: i64 = + (MONTH_LENGTH_DIVISOR * day_of_year + MARCH_DAY_ROUNDING) / MARCH_BASED_MONTH_DAYS; + let day: i64 = day_of_year + - (MARCH_BASED_MONTH_DAYS * shifted_month + MARCH_DAY_ROUNDING) / MONTH_LENGTH_DIVISOR + + 1; + let month: i64 = if shifted_month < FIRST_NEXT_YEAR_INDEX { + shifted_month + MARCH_OFFSET + } else { + shifted_month - MONTHS_FROM_MARCH + }; + let year: i64 = year_of_era + era * LEAP_CYCLE_YEARS + i64::from(month < i64::from(MARCH)); + Ok(CivilDate { + year, + month: u32::try_from(month).map_err(|_| overflow(DATE_OUT_OF_RANGE))?, + day: u32::try_from(day).map_err(|_| overflow(DATE_OUT_OF_RANGE))?, + }) +} + +/// Midnight at the start of `date`. +pub(crate) fn midnight(date: CivilDate) -> LocalDateTime { + LocalDateTime { + date, + hour: 0, + minute: 0, + second: 0, + microsecond: 0, + } +} + +fn micros_of_day(local: &LocalDateTime) -> i128 { + (i128::from(local.hour) * SECONDS_PER_HOUR + + i128::from(local.minute) * SECONDS_PER_MINUTE + + i128::from(local.second)) + * MICROS_PER_SECOND + + i128::from(local.microsecond) +} + +/// `date + timedelta(days=days)` for an already valid day count. +pub(crate) fn add_days(date: CivilDate, days: i128) -> Result { + from_ordinal(i128::from(ordinal(date)) + days) +} + +/// `datetime + timedelta(microseconds=delta)` for an already valid delta. +pub(crate) fn add_micros(local: &LocalDateTime, delta: i128) -> Result { + let total: i128 = micros_of_day(local) + delta; + let date: CivilDate = add_days(local.date, total.div_euclid(MICROS_PER_DAY))?; + let remainder: i128 = total.rem_euclid(MICROS_PER_DAY); + let seconds: i128 = remainder / MICROS_PER_SECOND; + Ok(LocalDateTime { + date, + hour: (seconds / SECONDS_PER_HOUR) as u32, + minute: (seconds % SECONDS_PER_HOUR / SECONDS_PER_MINUTE) as u32, + second: (seconds % SECONDS_PER_MINUTE) as u32, + microsecond: (remainder % MICROS_PER_SECOND) as u32, + }) +} + +/// The microseconds of a `timedelta` built from `total`, failing as its constructor does. +pub(crate) fn timedelta_micros(total: i128) -> Result { + let days: i128 = total.div_euclid(MICROS_PER_DAY); + if !(MIN_C_INT..=MAX_C_INT).contains(&days) { + return Err(overflow(C_INT_OVERFLOW)); + } + if days.abs() > MAX_TIMEDELTA_DAYS { + return Err(CursorError::Overflow(format!( + "days={days}; must have magnitude <= {MAX_TIMEDELTA_DAYS}" + ))); + } + Ok(total) +} + +/// The whole days of a valid `timedelta` of `total` microseconds. +pub(crate) fn timedelta_days(total: i128) -> i128 { + total.div_euclid(MICROS_PER_DAY) +} diff --git a/crates/sqlbuild-cursor/src/algebra/_helpers/comparison.rs b/crates/sqlbuild-cursor/src/algebra/_helpers/comparison.rs new file mode 100644 index 0000000000..59e410e04a --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/_helpers/comparison.rs @@ -0,0 +1,62 @@ +//! Cursor values projected into one sortable domain and compared there. + +use std::cmp::Ordering; + +use crate::algebra::_helpers::civil::{add_micros, midnight}; +use crate::algebra::_helpers::failures::algebra; +use crate::algebra::constants::{INTEGER_KEY_TYPE, MIXED_COMPARISON, MOMENT_KEY_TYPE}; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{Scalar, ScalarValue, SortKey, ZoneOffset}; +use crate::algebra::types::ZoneRules; + +/// An integer, a date's midnight, or a timestamp's naive UTC wall time. +pub(crate) fn sort_key(value: &Scalar, rules: &dyn ZoneRules) -> Result { + Ok(match &value.value { + ScalarValue::Integer(number) => SortKey::Integer(*number), + ScalarValue::Date(date) => SortKey::Moment(midnight(*date)), + ScalarValue::Timestamp(timestamp) => SortKey::Moment(match timestamp.offset { + ZoneOffset::Naive => timestamp.local, + ZoneOffset::Fixed(offset) => add_micros(×tamp.local, -i128::from(offset))?, + ZoneOffset::External(template) => { + rules.to_utc(template, ×tamp.local, timestamp.fold)? + } + }), + }) +} + +/// The ordering of two keys of one kind, or `None` across kinds. +pub(crate) fn key_ordering(left: &SortKey, right: &SortKey) -> Option { + match (left, right) { + (SortKey::Integer(left), SortKey::Integer(right)) => Some(left.cmp(right)), + (SortKey::Moment(left), SortKey::Moment(right)) => Some(left.cmp(right)), + _ => None, + } +} + +/// The Python type name of a key, as a failed rich comparison reports it. +pub(crate) fn key_type(key: &SortKey) -> &'static str { + match key { + SortKey::Integer(_) => INTEGER_KEY_TYPE, + SortKey::Moment(_) => MOMENT_KEY_TYPE, + } +} + +/// `TypeError` for `left right` across key kinds. +pub(crate) fn unorderable(operator: &str, left: &SortKey, right: &SortKey) -> CursorError { + CursorError::Type(format!( + "'{operator}' not supported between instances of '{}' and '{}'", + key_type(left), + key_type(right) + )) +} + +/// The ordering of two compatible cursor values. +pub(crate) fn compare_scalars( + left: &Scalar, + right: &Scalar, + rules: &dyn ZoneRules, +) -> Result { + let left_key: SortKey = sort_key(left, rules)?; + let right_key: SortKey = sort_key(right, rules)?; + key_ordering(&left_key, &right_key).ok_or_else(|| algebra(MIXED_COMPARISON)) +} diff --git a/crates/sqlbuild-cursor/src/algebra/_helpers/failures.rs b/crates/sqlbuild-cursor/src/algebra/_helpers/failures.rs new file mode 100644 index 0000000000..853a398bbc --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/_helpers/failures.rs @@ -0,0 +1,11 @@ +//! Cursor errors built from fixed messages. + +use crate::algebra::errors::CursorError; + +pub(crate) fn algebra(message: &str) -> CursorError { + CursorError::Algebra(message.to_owned()) +} + +pub(crate) fn overflow(message: &str) -> CursorError { + CursorError::Overflow(message.to_owned()) +} diff --git a/crates/sqlbuild-cursor/src/algebra/_helpers/intervals.rs b/crates/sqlbuild-cursor/src/algebra/_helpers/intervals.rs new file mode 100644 index 0000000000..369075af98 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/_helpers/intervals.rs @@ -0,0 +1,151 @@ +//! Half-open interval construction, intersection, merging and batch splitting. + +use std::cmp::Ordering; + +use crate::algebra::_helpers::arithmetic::{ + advance_scalar, advance_scalar_by, floor_scalar, fresh, integer_sum, +}; +use crate::algebra::_helpers::civil::midnight; +use crate::algebra::_helpers::comparison::{compare_scalars, key_ordering, sort_key, unorderable}; +use crate::algebra::_helpers::failures::algebra; +use crate::algebra::constants::{NON_POSITIVE_STEP, TEMPORAL_PARTITIONS_NEED_GRAIN}; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{ + Grain, Interval, Scalar, ScalarValue, SortKey, Timestamp, ZoneOffset, +}; +use crate::algebra::types::ZoneRules; + +/// The canonical partition containing an observed value. +pub(crate) fn observed_partition_value( + value: &Scalar, + grain: Option, +) -> Result { + if let ScalarValue::Integer(number) = &value.value { + let end: Scalar = fresh(ScalarValue::Integer(integer_sum(*number, 1)?)); + return Ok(Interval { + start: value.clone(), + end, + grain: None, + }); + } + let Some(grain) = grain else { + return Err(algebra(TEMPORAL_PARTITIONS_NEED_GRAIN)); + }; + let observed: Scalar = match &value.value { + ScalarValue::Date(date) if grain <= Grain::Hour => { + fresh(ScalarValue::Timestamp(Timestamp { + local: midnight(*date), + fold: false, + offset: ZoneOffset::Naive, + template: None, + source_text: None, + })) + } + _ => value.clone(), + }; + let start: Scalar = floor_scalar(&observed, Some(grain)); + let end: Scalar = advance_scalar(&start, Some(grain))?; + Ok(Interval { + start, + end, + grain: Some(grain), + }) +} + +/// The intersection of two intervals, if it is not empty. +pub(crate) fn clamp_interval( + interval: &Interval, + bounds: &Interval, + rules: &dyn ZoneRules, +) -> Result, CursorError> { + let start: &Scalar = if compare_scalars(&interval.start, &bounds.start, rules)?.is_ge() { + &interval.start + } else { + &bounds.start + }; + let end: &Scalar = if compare_scalars(&interval.end, &bounds.end, rules)?.is_le() { + &interval.end + } else { + &bounds.end + }; + if compare_scalars(start, end, rules)?.is_ge() { + return Ok(None); + } + Ok(Some(Interval { + start: start.clone(), + end: end.clone(), + grain: interval.grain, + })) +} + +/// The intervals stably ordered by start, as `sorted` orders them. +fn ordered_by_start( + intervals: &[Interval], + rules: &dyn ZoneRules, +) -> Result, CursorError> { + let keys: Vec = intervals + .iter() + .map(|interval| sort_key(&interval.start, rules)) + .collect::>()?; + if let Some(pair) = keys + .windows(2) + .find(|pair| key_ordering(&pair[1], &pair[0]).is_none()) + { + return Err(unorderable("<", &pair[1], &pair[0])); + } + let mut order: Vec = (0..intervals.len()).collect(); + order.sort_by(|left, right| { + key_ordering(&keys[*left], &keys[*right]).unwrap_or(Ordering::Equal) + }); + Ok(order + .into_iter() + .map(|index| intervals[index].clone()) + .collect()) +} + +/// Overlapping or adjacent intervals merged, in start order. +pub(crate) fn merge_interval_values( + intervals: &[Interval], + rules: &dyn ZoneRules, +) -> Result, CursorError> { + let mut merged: Vec = Vec::new(); + for interval in ordered_by_start(intervals, rules)? { + let Some(last) = merged.last_mut() else { + merged.push(interval); + continue; + }; + if compare_scalars(&last.end, &interval.start, rules)?.is_lt() { + merged.push(interval); + } else if compare_scalars(&last.end, &interval.end, rules)?.is_lt() { + last.end = interval.end; + last.grain = interval.grain; + } + } + Ok(merged) +} + +/// Consecutive batches of `step` grains covering `interval`, the last one possibly shorter. +pub(crate) fn split_interval( + interval: &Interval, + step: i64, + rules: &dyn ZoneRules, +) -> Result, CursorError> { + if step <= 0 { + return Err(algebra(NON_POSITIVE_STEP)); + } + let mut batches: Vec = Vec::new(); + let mut start: Scalar = interval.start.clone(); + while compare_scalars(&start, &interval.end, rules)?.is_lt() { + let mut end: Scalar = advance_scalar_by(&start, interval.grain, step)?; + if compare_scalars(&end, &interval.end, rules)?.is_gt() { + end = interval.end.clone(); + } + batches.push(Interval { + start, + end: end.clone(), + grain: interval.grain, + }); + start = end; + } + Ok(batches) +} diff --git a/crates/sqlbuild-cursor/src/algebra/_helpers/mod.rs b/crates/sqlbuild-cursor/src/algebra/_helpers/mod.rs new file mode 100644 index 0000000000..a747370813 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod arithmetic; +pub(crate) mod civil; +pub(crate) mod comparison; +pub(crate) mod failures; +pub(crate) mod intervals; diff --git a/crates/sqlbuild-cursor/src/algebra/constants.rs b/crates/sqlbuild-cursor/src/algebra/constants.rs new file mode 100644 index 0000000000..1216a56a02 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/constants.rs @@ -0,0 +1,48 @@ +//! Calendar sizes and Python's `datetime` limits the cursor algebra reproduces. + +pub(crate) const MONTHS_PER_YEAR: i64 = 12; +pub(crate) const MIN_YEAR: i64 = 1; +pub(crate) const MAX_YEAR: i64 = 9999; +pub(crate) const MICROS_PER_SECOND: i128 = 1_000_000; +pub(crate) const SECONDS_PER_MINUTE: i128 = 60; +pub(crate) const SECONDS_PER_HOUR: i128 = 3_600; +pub(crate) const SECONDS_PER_DAY: i128 = 86_400; +pub(crate) const MICROS_PER_DAY: i128 = SECONDS_PER_DAY * MICROS_PER_SECOND; +/// `timedelta.max.days`. +pub(crate) const MAX_TIMEDELTA_DAYS: i128 = 999_999_999; +/// The largest C `int` a `timedelta` day count passes through. +pub(crate) const MAX_C_INT: i128 = i32::MAX as i128; +pub(crate) const MIN_C_INT: i128 = i32::MIN as i128; +pub(crate) const FEBRUARY: u32 = 2; +pub(crate) const DAYS_IN_MONTH: [u32; 12] = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; +pub(crate) const LEAP_CYCLE_YEARS: i64 = 400; +pub(crate) const CENTURY_YEARS: i64 = 100; +pub(crate) const LEAP_YEARS: i64 = 4; + +pub const DATE_OUT_OF_RANGE: &str = "date value out of range"; +pub const DAY_OUT_OF_RANGE: &str = "day is out of range for month"; +pub const C_INT_OVERFLOW: &str = "Python int too large to convert to C int"; +pub const INTEGER_CURSOR_OVERFLOW: &str = "integer cursor value out of range"; +pub const NON_POSITIVE_STEP: &str = "cursor batch step must be positive"; +pub const TEMPORAL_VALUES_NEED_GRAIN: &str = "temporal cursor values require a grain"; +pub const TEMPORAL_PARTITIONS_NEED_GRAIN: &str = "temporal observed partitions require a grain"; +pub const TEMPORAL_BOUNDS_NEED_GRAIN: &str = "temporal inclusive bounds require a grain"; +pub const TEMPORAL_INTERVALS_NEED_GRAIN: &str = "temporal cursor intervals require a grain"; +pub const MIXED_COMPARISON: &str = "cannot compare temporal and integer cursor values"; +pub const START_NOT_BEFORE_END: &str = "cursor interval start must be less than end"; +pub const START_NOT_ALIGNED: &str = "cursor interval start must be grain-aligned"; +pub const END_NOT_ALIGNED: &str = "cursor interval end must be grain-aligned"; +pub const INTEGER_KEY_TYPE: &str = "int"; +pub const MOMENT_KEY_TYPE: &str = "datetime.datetime"; +pub const NO_GRAIN_STEP: &str = "a timestamp cursor value needs a grain to advance"; +pub(crate) const MARCH: u32 = 3; +/// The first month index from March on that falls in the next calendar year. +pub(crate) const FIRST_NEXT_YEAR_INDEX: i64 = 10; +pub(crate) const UTC_DESIGNATOR: char = 'Z'; +pub(crate) const OFFSET_SEPARATOR: char = ':'; +pub(crate) const COMPACT_OFFSET_LENGTH: usize = 5; +pub(crate) const SEPARATED_OFFSET_LENGTH: usize = 6; +pub(crate) const SEPARATOR_POSITION: usize = 3; +/// Unit suffixes in the order a duration string must name them. +pub(crate) const UNIT_SUFFIXES: [&str; 6] = ["y", "mo", "d", "h", "m", "s"]; +pub(crate) const LINE_FEED: char = '\n'; diff --git a/crates/sqlbuild-cursor/src/algebra/errors.rs b/crates/sqlbuild-cursor/src/algebra/errors.rs new file mode 100644 index 0000000000..f7b1b85f51 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/errors.rs @@ -0,0 +1,29 @@ +//! Cursor algebra failures, each the exception Python raises for the same input. + +use std::fmt::{self, Display}; + +/// A failed cursor operation and the Python exception class that reports it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CursorError { + /// `CursorAlgebraError`: values that cannot form a valid typed interval. + Algebra(String), + /// `OverflowError` from `datetime` or `timedelta` arithmetic. + Overflow(String), + /// `ValueError` from `date.replace`. + Value(String), + /// `TypeError` from ordering keys of different kinds. + Type(String), +} + +impl Display for CursorError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Algebra(message) + | Self::Overflow(message) + | Self::Value(message) + | Self::Type(message) => message, + }) + } +} + +impl std::error::Error for CursorError {} diff --git a/crates/sqlbuild-cursor/src/algebra/main/cap_from_end.rs b/crates/sqlbuild-cursor/src/algebra/main/cap_from_end.rs new file mode 100644 index 0000000000..019b9f9388 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/cap_from_end.rs @@ -0,0 +1,10 @@ +//! `cap_from_end`: the trailing intervals a batch cap keeps. + +use std::ops::Range; + +/// `items[-count:] if count > 0 else ()` as a range over `length` items. +#[must_use] +pub fn cap_from_end(length: usize, count: i64) -> Range { + let kept: usize = usize::try_from(count).unwrap_or(0).min(length); + length - kept..length +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/cap_from_start.rs b/crates/sqlbuild-cursor/src/algebra/main/cap_from_start.rs new file mode 100644 index 0000000000..c1375fa2a0 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/cap_from_start.rs @@ -0,0 +1,9 @@ +//! `cap_from_start`: the leading intervals a batch cap keeps. + +use std::ops::Range; + +/// `items[:max(count, 0)]` as a range over `length` items. +#[must_use] +pub fn cap_from_start(length: usize, count: i64) -> Range { + 0..usize::try_from(count).unwrap_or(0).min(length) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/clamp.rs b/crates/sqlbuild-cursor/src/algebra/main/clamp.rs new file mode 100644 index 0000000000..aef4997d7e --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/clamp.rs @@ -0,0 +1,15 @@ +//! `clamp`: the intersection of two half-open cursor intervals. + +use crate::algebra::_helpers::intervals::clamp_interval; +use crate::algebra::errors::CursorError; +use crate::algebra::models::Interval; +use crate::algebra::types::ZoneRules; + +/// `interval` limited to `bounds`, or `None` when they do not overlap. +pub fn clamp( + interval: &Interval, + bounds: &Interval, + rules: &dyn ZoneRules, +) -> Result, CursorError> { + clamp_interval(interval, bounds, rules) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/compare.rs b/crates/sqlbuild-cursor/src/algebra/main/compare.rs new file mode 100644 index 0000000000..bf3f170009 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/compare.rs @@ -0,0 +1,17 @@ +//! `compare`: the ordering of two compatible cursor values. + +use std::cmp::Ordering; + +use crate::algebra::_helpers::comparison::compare_scalars; +use crate::algebra::errors::CursorError; +use crate::algebra::models::Scalar; +use crate::algebra::types::ZoneRules; + +/// How `left` orders against `right`; integers and temporal values do not compare. +pub fn compare( + left: &Scalar, + right: &Scalar, + rules: &dyn ZoneRules, +) -> Result { + compare_scalars(left, right, rules) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/cursor_sort_key.rs b/crates/sqlbuild-cursor/src/algebra/main/cursor_sort_key.rs new file mode 100644 index 0000000000..8495bcd1b7 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/cursor_sort_key.rs @@ -0,0 +1,11 @@ +//! `cursor_sort_key`: the sortable projection of one cursor value. + +use crate::algebra::_helpers::comparison::sort_key; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{Scalar, SortKey}; +use crate::algebra::types::ZoneRules; + +/// The integer, or the naive UTC wall time, `value` sorts by. +pub fn cursor_sort_key(value: &Scalar, rules: &dyn ZoneRules) -> Result { + sort_key(value, rules) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/exclusive_end_from_observed_max.rs b/crates/sqlbuild-cursor/src/algebra/main/exclusive_end_from_observed_max.rs new file mode 100644 index 0000000000..57acf2e5ee --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/exclusive_end_from_observed_max.rs @@ -0,0 +1,18 @@ +//! `exclusive_end_from_observed_max`: availability derived from an observed cursor maximum. + +use crate::algebra::_helpers::intervals::observed_partition_value; +use crate::algebra::errors::CursorError; +use crate::algebra::main::validate_interval::validate_interval; +use crate::algebra::models::{Grain, Interval, Scalar}; +use crate::algebra::types::ZoneRules; + +/// The end of the validated partition holding the observed maximum `value`. +pub fn exclusive_end_from_observed_max( + value: &Scalar, + grain: Option, + rules: &dyn ZoneRules, +) -> Result { + let partition: Interval = observed_partition_value(value, grain)?; + validate_interval(&partition, rules)?; + Ok(partition.end) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/exclusive_to_inclusive.rs b/crates/sqlbuild-cursor/src/algebra/main/exclusive_to_inclusive.rs new file mode 100644 index 0000000000..7c39e279b6 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/exclusive_to_inclusive.rs @@ -0,0 +1,11 @@ +//! `exclusive_to_inclusive`: an exclusive bound as an inclusive one. + +use crate::algebra::_helpers::arithmetic::shift_bound; +use crate::algebra::constants::TEMPORAL_BOUNDS_NEED_GRAIN; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{Grain, Scalar}; + +/// `value` moved back by one cursor unit: one, a day, or the grain's fixed step. +pub fn exclusive_to_inclusive(value: &Scalar, grain: Option) -> Result { + shift_bound(value, grain, -1, TEMPORAL_BOUNDS_NEED_GRAIN) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/extreme_position.rs b/crates/sqlbuild-cursor/src/algebra/main/extreme_position.rs new file mode 100644 index 0000000000..139db90344 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/extreme_position.rs @@ -0,0 +1,30 @@ +//! `extreme_position`: which key `max()` or `min()` selects. + +use std::cmp::Ordering; + +use crate::algebra::_helpers::comparison::{key_ordering, unorderable}; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{Extreme, SortKey}; + +/// The first position holding the largest or smallest key, failing as `max`/`min` fail. +pub fn extreme_position(keys: &[SortKey], extreme: Extreme) -> Result { + let (operator, wanted, name): (&str, Ordering, &str) = match extreme { + Extreme::Max => (">", Ordering::Greater, "max"), + Extreme::Min => ("<", Ordering::Less, "min"), + }; + let Some(first) = keys.first() else { + return Err(CursorError::Value(format!( + "{name}() iterable argument is empty" + ))); + }; + let mut best: (usize, &SortKey) = (0, first); + for (index, key) in keys.iter().enumerate().skip(1) { + let Some(ordering) = key_ordering(key, best.1) else { + return Err(unorderable(operator, key, best.1)); + }; + if ordering == wanted { + best = (index, key); + } + } + Ok(best.0) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/floor_to_grain.rs b/crates/sqlbuild-cursor/src/algebra/main/floor_to_grain.rs new file mode 100644 index 0000000000..4a15041e7d --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/floor_to_grain.rs @@ -0,0 +1,10 @@ +//! `floor_to_grain`: the aligned boundary at or before a cursor value. + +use crate::algebra::_helpers::arithmetic::floor_scalar; +use crate::algebra::models::{Grain, Scalar}; + +/// `value` floored to `grain`; integers and day-or-finer dates come back unchanged. +#[must_use] +pub fn floor_to_grain(value: &Scalar, grain: Option) -> Scalar { + floor_scalar(value, grain) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/inclusive_to_exclusive.rs b/crates/sqlbuild-cursor/src/algebra/main/inclusive_to_exclusive.rs new file mode 100644 index 0000000000..378481ca27 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/inclusive_to_exclusive.rs @@ -0,0 +1,11 @@ +//! `inclusive_to_exclusive`: an operator-declared inclusive bound as an exclusive one. + +use crate::algebra::_helpers::arithmetic::shift_bound; +use crate::algebra::constants::TEMPORAL_BOUNDS_NEED_GRAIN; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{Grain, Scalar}; + +/// `value` advanced by one cursor unit: one, a day, or the grain's fixed step. +pub fn inclusive_to_exclusive(value: &Scalar, grain: Option) -> Result { + shift_bound(value, grain, 1, TEMPORAL_BOUNDS_NEED_GRAIN) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/merge.rs b/crates/sqlbuild-cursor/src/algebra/main/merge.rs new file mode 100644 index 0000000000..6ac602f319 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/merge.rs @@ -0,0 +1,11 @@ +//! `merge`: overlapping or adjacent cursor intervals joined. + +use crate::algebra::_helpers::intervals::merge_interval_values; +use crate::algebra::errors::CursorError; +use crate::algebra::models::Interval; +use crate::algebra::types::ZoneRules; + +/// The intervals ordered by start with every overlapping or touching run joined. +pub fn merge(intervals: &[Interval], rules: &dyn ZoneRules) -> Result, CursorError> { + merge_interval_values(intervals, rules) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/mod.rs b/crates/sqlbuild-cursor/src/algebra/main/mod.rs new file mode 100644 index 0000000000..5b56034381 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/mod.rs @@ -0,0 +1,20 @@ +pub mod cap_from_end; +pub mod cap_from_start; +pub mod clamp; +pub mod compare; +pub mod cursor_sort_key; +pub mod exclusive_end_from_observed_max; +pub mod exclusive_to_inclusive; +pub mod extreme_position; +pub mod floor_to_grain; +pub mod inclusive_to_exclusive; +pub mod merge; +pub mod next_boundary; +pub mod normalize_timestamp_text; +pub mod observed_partition; +pub mod parse_duration; +pub mod parse_integer_text; +pub mod render; +pub mod shift_by_duration; +pub mod split_into_batches; +pub mod validate_interval; diff --git a/crates/sqlbuild-cursor/src/algebra/main/next_boundary.rs b/crates/sqlbuild-cursor/src/algebra/main/next_boundary.rs new file mode 100644 index 0000000000..08a0b8a0e2 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/next_boundary.rs @@ -0,0 +1,10 @@ +//! `next_boundary`: the boundary one grain after an aligned cursor value. + +use crate::algebra::_helpers::arithmetic::advance_scalar; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{Grain, Scalar}; + +/// `value` advanced by one `grain`. +pub fn next_boundary(value: &Scalar, grain: Option) -> Result { + advance_scalar(value, grain) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/normalize_timestamp_text.rs b/crates/sqlbuild-cursor/src/algebra/main/normalize_timestamp_text.rs new file mode 100644 index 0000000000..a810f9acd1 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/normalize_timestamp_text.rs @@ -0,0 +1,58 @@ +//! `normalize_timestamp_text`: warehouse timestamp text in an ISO-compatible spelling. + +use sqlbuild_core::text::main::is_decimal_digit::is_decimal_digit; +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::algebra::constants::{ + COMPACT_OFFSET_LENGTH, OFFSET_SEPARATOR, SEPARATED_OFFSET_LENGTH, SEPARATOR_POSITION, + UTC_DESIGNATOR, +}; + +/// Whether `tail` is a `[+-]dd:?dd` offset with Unicode decimal digits. +fn is_offset(tail: &[char]) -> bool { + let Some((sign, rest)) = tail.split_first() else { + return false; + }; + let digits: Vec = match rest.len() + 1 { + COMPACT_OFFSET_LENGTH => rest.to_vec(), + SEPARATED_OFFSET_LENGTH if rest[SEPARATOR_POSITION - 1] == OFFSET_SEPARATOR => rest + .iter() + .enumerate() + .filter(|(index, _)| *index != SEPARATOR_POSITION - 1) + .map(|(_, character)| *character) + .collect(), + _ => return false, + }; + matches!(sign, '+' | '-') && digits.into_iter().all(is_decimal_digit) +} + +/// The length of the UTC designator or offset ending `characters`, if one does. +fn zone_suffix_length(characters: &[char]) -> Option { + if characters.last() == Some(&UTC_DESIGNATOR) { + return Some(1); + } + [COMPACT_OFFSET_LENGTH, SEPARATED_OFFSET_LENGTH] + .into_iter() + .find(|length| { + characters.len() >= *length && is_offset(&characters[characters.len() - length..]) + }) +} + +/// `re.sub(r"\s+(?=(?:Z|[+-]\d{2}:?\d{2})$)", "", value.strip())`. +#[must_use] +pub fn normalize_timestamp_text(value: &str) -> String { + let characters: Vec = python_strip(value).chars().collect(); + let Some(suffix) = zone_suffix_length(&characters) else { + return characters.into_iter().collect(); + }; + let zone_start: usize = characters.len() - suffix; + let mut space_start: usize = zone_start; + while space_start > 0 && python_isspace(characters[space_start - 1]) { + space_start -= 1; + } + characters[..space_start] + .iter() + .chain(&characters[zone_start..]) + .collect() +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/observed_partition.rs b/crates/sqlbuild-cursor/src/algebra/main/observed_partition.rs new file mode 100644 index 0000000000..e4db7c9199 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/observed_partition.rs @@ -0,0 +1,10 @@ +//! `observed_partition`: the canonical partition containing an observed cursor value. + +use crate::algebra::_helpers::intervals::observed_partition_value; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{Grain, Interval, Scalar}; + +/// The grain-wide interval holding `value`; an integer's partition is the next integer. +pub fn observed_partition(value: &Scalar, grain: Option) -> Result { + observed_partition_value(value, grain) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/parse_duration.rs b/crates/sqlbuild-cursor/src/algebra/main/parse_duration.rs new file mode 100644 index 0000000000..385ea6f71f --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/parse_duration.rs @@ -0,0 +1,57 @@ +//! `parse_duration`: the units of a duration string like `1y6mo`, `30d` or `6h`. + +use sqlbuild_core::text::main::decimal_digit_value::decimal_digit_value; + +use crate::algebra::constants::{LINE_FEED, UNIT_SUFFIXES}; +use crate::algebra::models::DurationDigits; + +/// The ASCII digits of one `(\d+)` group at the start of `characters`, and its length. +fn unit_group(characters: &[char], suffix: &str) -> Option<(String, usize)> { + let digits: String = characters + .iter() + .map_while(|character| decimal_digit_value(*character)) + .map(|digit| char::from(b'0' + digit as u8)) + .collect(); + let count: usize = digits.chars().count(); + let suffix_characters: Vec = suffix.chars().collect(); + let matched: bool = count > 0 && characters[count..].starts_with(&suffix_characters); + matched.then(|| (digits, count + suffix_characters.len())) +} + +fn is_zero(digits: &str) -> bool { + digits.bytes().all(|byte| byte == b'0') +} + +/// Each unit's digits, `"0"` when absent, if `value` fully matches and is not zero. +#[must_use] +pub fn parse_duration(value: &str) -> Option { + let characters: Vec = value.chars().collect(); + let mut position: usize = 0; + let mut groups: Vec = Vec::with_capacity(UNIT_SUFFIXES.len()); + for suffix in UNIT_SUFFIXES { + match unit_group(&characters[position..], suffix) { + Some((digits, length)) => { + groups.push(digits); + position += length; + } + None => groups.push(String::from("0")), + } + } + let rest: &[char] = &characters[position..]; + if !(rest.is_empty() || rest == [LINE_FEED]) { + return None; + } + if groups.iter().all(|digits| is_zero(digits)) { + return None; + } + let mut parts = groups.into_iter(); + let mut next = || parts.next().unwrap_or_default(); + Some(DurationDigits { + years: next(), + months: next(), + days: next(), + hours: next(), + minutes: next(), + seconds: next(), + }) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/parse_integer_text.rs b/crates/sqlbuild-cursor/src/algebra/main/parse_integer_text.rs new file mode 100644 index 0000000000..bce5763fc9 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/parse_integer_text.rs @@ -0,0 +1,14 @@ +//! `parse_integer_text`: the fast path of integer cursor text. + +/// The value of `text` when it is a plain ASCII signed integer that fits `i128`. +#[must_use] +pub fn parse_integer_text(text: &str) -> Option { + let digits: &str = text.strip_prefix(['+', '-']).unwrap_or(text); + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let Ok(value) = text.parse::() else { + return None; + }; + Some(value) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/render.rs b/crates/sqlbuild-cursor/src/algebra/main/render.rs new file mode 100644 index 0000000000..cfb43c235d --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/render.rs @@ -0,0 +1,62 @@ +//! `render`: a cursor value in the cursor output format. + +use crate::algebra::constants::{MICROS_PER_SECOND, SECONDS_PER_HOUR, SECONDS_PER_MINUTE}; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{CivilDate, Scalar, ScalarValue, ZoneOffset}; +use crate::algebra::types::ZoneRules; + +fn date_text(date: &CivilDate) -> String { + format!("{:04}-{:02}-{:02}", date.year, date.month, date.day) +} + +/// `timedelta` offset text as `datetime.isoformat` appends it. +fn offset_text(offset: i64) -> String { + let sign: char = if offset < 0 { '-' } else { '+' }; + let magnitude: i128 = i128::from(offset).abs(); + let seconds: i128 = magnitude / MICROS_PER_SECOND; + let microseconds: i128 = magnitude % MICROS_PER_SECOND; + let mut text: String = format!( + "{sign}{:02}:{:02}", + seconds / SECONDS_PER_HOUR, + seconds % SECONDS_PER_HOUR / SECONDS_PER_MINUTE + ); + if seconds % SECONDS_PER_MINUTE != 0 || microseconds != 0 { + text.push_str(&format!(":{:02}", seconds % SECONDS_PER_MINUTE)); + } + if microseconds != 0 { + text.push_str(&format!(".{microseconds:06}")); + } + text +} + +/// The integer's digits, the parsed timestamp text, or the value's `isoformat()`. +pub fn render(value: &Scalar, rules: &dyn ZoneRules) -> Result { + let timestamp = match &value.value { + ScalarValue::Integer(number) => return Ok(number.to_string()), + ScalarValue::Date(date) => return Ok(date_text(date)), + ScalarValue::Timestamp(timestamp) => timestamp, + }; + if let Some(text) = ×tamp.source_text { + return Ok(text.clone()); + } + let local = ×tamp.local; + let mut text: String = format!( + "{}T{:02}:{:02}:{:02}", + date_text(&local.date), + local.hour, + local.minute, + local.second + ); + if local.microsecond != 0 { + text.push_str(&format!(".{:06}", local.microsecond)); + } + let offset: Option = match timestamp.offset { + ZoneOffset::Naive => None, + ZoneOffset::Fixed(offset) => Some(offset), + ZoneOffset::External(template) => rules.utc_offset(template, local, timestamp.fold)?, + }; + if let Some(offset) = offset { + text.push_str(&offset_text(offset)); + } + Ok(text) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/shift_by_duration.rs b/crates/sqlbuild-cursor/src/algebra/main/shift_by_duration.rs new file mode 100644 index 0000000000..7d72b7b683 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/shift_by_duration.rs @@ -0,0 +1,50 @@ +//! `shift_by_duration`: a date or timestamp moved by a calendar duration. + +use crate::algebra::_helpers::civil::{ + add_days, add_micros, checked_date, days_in_month, timedelta_days, timedelta_micros, +}; +use crate::algebra::constants::{ + MICROS_PER_SECOND, MONTHS_PER_YEAR, SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE, +}; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{CivilDate, DurationSpan, Moment}; + +/// `moment.replace(year, month, day)` `delta` months on, with the day clamped to the month. +fn shift_months(date: CivilDate, delta: i64) -> Result { + let total: i64 = date.year * MONTHS_PER_YEAR + i64::from(date.month) - 1 + delta; + let year: i64 = total.div_euclid(MONTHS_PER_YEAR); + let month: u32 = (total.rem_euclid(MONTHS_PER_YEAR) + 1) as u32; + checked_date(year, month, date.day.min(days_in_month(year, month))) +} + +/// `Duration.add_to` for a positive `direction`, `Duration.subtract_from` for a negative one. +pub fn shift_by_duration( + moment: Moment, + span: &DurationSpan, + direction: i64, +) -> Result { + let months: i64 = span.months * direction; + let shifted: Moment = match moment { + _ if months == 0 => moment, + Moment::Date(date) => Moment::Date(shift_months(date, months)?), + Moment::Timestamp { mut local, fold } => { + local.date = shift_months(local.date, months)?; + Moment::Timestamp { local, fold } + } + }; + let fixed: i128 = timedelta_micros( + (i128::from(span.days) * SECONDS_PER_DAY + + i128::from(span.hours) * SECONDS_PER_HOUR + + i128::from(span.minutes) * SECONDS_PER_MINUTE + + i128::from(span.seconds)) + * MICROS_PER_SECOND, + )?; + let sign: i128 = i128::from(direction.signum()); + Ok(match shifted { + Moment::Date(date) => Moment::Date(add_days(date, sign * timedelta_days(fixed))?), + Moment::Timestamp { local, .. } => Moment::Timestamp { + local: add_micros(&local, sign * fixed)?, + fold: false, + }, + }) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/split_into_batches.rs b/crates/sqlbuild-cursor/src/algebra/main/split_into_batches.rs new file mode 100644 index 0000000000..1be5f05953 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/split_into_batches.rs @@ -0,0 +1,15 @@ +//! `split_into_batches`: one cursor interval cut into batches. + +use crate::algebra::_helpers::intervals::split_interval; +use crate::algebra::errors::CursorError; +use crate::algebra::models::Interval; +use crate::algebra::types::ZoneRules; + +/// Consecutive batches of a positive number of grains; the last ends at the interval's end. +pub fn split_into_batches( + interval: &Interval, + step: i64, + rules: &dyn ZoneRules, +) -> Result, CursorError> { + split_interval(interval, step, rules) +} diff --git a/crates/sqlbuild-cursor/src/algebra/main/validate_interval.rs b/crates/sqlbuild-cursor/src/algebra/main/validate_interval.rs new file mode 100644 index 0000000000..361d0d678f --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/main/validate_interval.rs @@ -0,0 +1,42 @@ +//! `validate_interval`: the checks an aligned interval must pass when it is built. + +use crate::algebra::_helpers::arithmetic::floor_scalar; +use crate::algebra::_helpers::comparison::compare_scalars; +use crate::algebra::_helpers::failures::algebra; +use crate::algebra::constants::{ + END_NOT_ALIGNED, START_NOT_ALIGNED, START_NOT_BEFORE_END, TEMPORAL_INTERVALS_NEED_GRAIN, +}; +use crate::algebra::errors::CursorError; +use crate::algebra::models::{Interval, Scalar, ScalarValue}; +use crate::algebra::types::ZoneRules; + +/// Whether two values of one kind are equal as `datetime` equality in one zone reads them. +fn same_value(left: &Scalar, right: &Scalar) -> bool { + match (&left.value, &right.value) { + (ScalarValue::Timestamp(left), ScalarValue::Timestamp(right)) => left.local == right.local, + (left, right) => left == right, + } +} + +/// An ordered interval whose temporal ends carry a grain and sit on its boundaries. +pub fn validate_interval(interval: &Interval, rules: &dyn ZoneRules) -> Result<(), CursorError> { + if compare_scalars(&interval.start, &interval.end, rules)?.is_ge() { + return Err(algebra(START_NOT_BEFORE_END)); + } + let Some(grain) = interval.grain else { + let integral: bool = matches!(interval.start.value, ScalarValue::Integer(_)) + && matches!(interval.end.value, ScalarValue::Integer(_)); + return if integral { + Ok(()) + } else { + Err(algebra(TEMPORAL_INTERVALS_NEED_GRAIN)) + }; + }; + if !same_value(&floor_scalar(&interval.start, Some(grain)), &interval.start) { + return Err(algebra(START_NOT_ALIGNED)); + } + if !same_value(&floor_scalar(&interval.end, Some(grain)), &interval.end) { + return Err(algebra(END_NOT_ALIGNED)); + } + Ok(()) +} diff --git a/crates/sqlbuild-cursor/src/algebra/mod.rs b/crates/sqlbuild-cursor/src/algebra/mod.rs new file mode 100644 index 0000000000..65aaca2009 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cursor/src/algebra/models.rs b/crates/sqlbuild-cursor/src/algebra/models.rs new file mode 100644 index 0000000000..b40dd713ec --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/models.rs @@ -0,0 +1,118 @@ +//! Cursor values, grains, intervals and durations. + +/// A proleptic Gregorian date; the year may leave `datetime`'s range only inside arithmetic. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct CivilDate { + pub year: i64, + pub month: u32, + pub day: u32, +} + +/// A naive wall-clock date and time with microsecond precision. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct LocalDateTime { + pub date: CivilDate, + pub hour: u32, + pub minute: u32, + pub second: u32, + pub microsecond: u32, +} + +/// How a timestamp's wall time relates to UTC. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ZoneOffset { + /// A naive timestamp, compared as written. + Naive, + /// A fixed `datetime.timezone` offset in microseconds. + Fixed(i64), + /// Any other `tzinfo`, resolved through the caller's `ZoneRules` for this input timestamp. + External(usize), +} + +/// A timestamp cursor value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Timestamp { + pub local: LocalDateTime, + pub fold: bool, + pub offset: ZoneOffset, + /// The caller's input timestamp whose type and `tzinfo` this value keeps. + pub template: Option, + /// The text the value was parsed from, rendered in place of `isoformat()`. + pub source_text: Option, +} + +/// The typed payload of a cursor value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ScalarValue { + Timestamp(Timestamp), + Date(CivilDate), + Integer(i128), +} + +/// A cursor value and, while it is unchanged, the caller's handle for the input it came from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Scalar { + pub value: ScalarValue, + pub handle: Option, +} + +/// A cursor grain, from finest to coarsest. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Grain { + Second, + Minute, + Hour, + Day, + Month, + Year, +} + +/// A half-open cursor interval. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Interval { + pub start: Scalar, + pub end: Scalar, + pub grain: Option, +} + +/// The sortable projection of a cursor value: an integer, or a naive UTC wall time. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SortKey { + Integer(i128), + Moment(LocalDateTime), +} + +/// The value a calendar duration moves: a date, or a timestamp's wall time and fold. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Moment { + Date(CivilDate), + Timestamp { local: LocalDateTime, fold: bool }, +} + +/// A parsed duration's whole-month and fixed parts, as Python integers would hold them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DurationSpan { + pub months: i64, + pub days: i64, + pub hours: i64, + pub minutes: i64, + pub seconds: i64, +} + +/// The ASCII digits of each unit a duration string names, years first. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DurationDigits { + pub years: String, + pub months: String, + pub days: String, + pub hours: String, + pub minutes: String, + pub seconds: String, +} + +/// Which end of an ordering `max()` or `min()` keeps. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Extreme { + Max, + Min, +} diff --git a/crates/sqlbuild-cursor/src/algebra/tests.rs b/crates/sqlbuild-cursor/src/algebra/tests.rs new file mode 100644 index 0000000000..2bf30698c2 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/tests.rs @@ -0,0 +1,10 @@ +#[path = "tests/test_durations.rs"] +mod durations; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_intervals.rs"] +mod intervals; +#[path = "tests/test_scalars.rs"] +mod scalars; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cursor/src/algebra/tests/helpers.rs b/crates/sqlbuild-cursor/src/algebra/tests/helpers.rs new file mode 100644 index 0000000000..18fd7dcfe4 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/tests/helpers.rs @@ -0,0 +1,72 @@ +use crate::algebra::errors::CursorError; +use crate::algebra::models::{ + CivilDate, LocalDateTime, Scalar, ScalarValue, Timestamp, ZoneOffset, +}; +use crate::algebra::tests::test_types::NoZones; +use crate::algebra::types::ZoneRules; + +impl ZoneRules for NoZones { + fn to_utc( + &self, + _: usize, + local: &LocalDateTime, + _: bool, + ) -> Result { + Ok(*local) + } + + fn utc_offset(&self, _: usize, _: &LocalDateTime, _: bool) -> Result, CursorError> { + Ok(None) + } +} + +pub(crate) fn civil(year: i64, month: u32, day: u32) -> CivilDate { + CivilDate { year, month, day } +} + +pub(crate) fn local( + date: CivilDate, + (hour, minute, second, microsecond): (u32, u32, u32, u32), +) -> LocalDateTime { + LocalDateTime { + date, + hour, + minute, + second, + microsecond, + } +} + +pub(crate) fn timestamp(local: LocalDateTime, offset: ZoneOffset) -> Scalar { + Scalar { + value: ScalarValue::Timestamp(Timestamp { + local, + fold: false, + offset, + template: None, + source_text: None, + }), + handle: None, + } +} + +pub(crate) fn naive(year: i64, month: u32, day: u32, hour: u32) -> Scalar { + timestamp( + local(civil(year, month, day), (hour, 0, 0, 0)), + ZoneOffset::Naive, + ) +} + +pub(crate) fn date(year: i64, month: u32, day: u32) -> Scalar { + Scalar { + value: ScalarValue::Date(civil(year, month, day)), + handle: None, + } +} + +pub(crate) fn integer(value: i128) -> Scalar { + Scalar { + value: ScalarValue::Integer(value), + handle: None, + } +} diff --git a/crates/sqlbuild-cursor/src/algebra/tests/test_durations.rs b/crates/sqlbuild-cursor/src/algebra/tests/test_durations.rs new file mode 100644 index 0000000000..3e94b84860 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/tests/test_durations.rs @@ -0,0 +1,115 @@ +use crate::algebra::errors::CursorError; +use crate::algebra::main::parse_duration::parse_duration; +use crate::algebra::main::shift_by_duration::shift_by_duration; +use crate::algebra::models::{DurationSpan, Moment}; +use crate::algebra::tests::helpers::{civil, local}; +use crate::algebra::tests::test_types::{DurationTestCase, ShiftTestCase}; + +const NO_SPAN: DurationSpan = DurationSpan { + months: 0, + days: 0, + hours: 0, + minutes: 0, + seconds: 0, +}; + +#[test] +fn given_duration_text_when_parsing_then_matches_the_python_pattern() { + let test_cases = [ + DurationTestCase { + description: "every unit in order", + text: "1y2mo3d4h5m6s", + expected_digits: Some(["1", "2", "3", "4", "5", "6"]), + }, + DurationTestCase { + description: "a trailing newline and Unicode digits", + text: "\u{0663}d\n", + expected_digits: Some(["0", "0", "3", "0", "0", "0"]), + }, + DurationTestCase { + description: "units out of order", + text: "1mo1y", + expected_digits: None, + }, + DurationTestCase { + description: "a zero duration", + text: "0y0mo", + expected_digits: None, + }, + ]; + + for test_case in test_cases { + let digits = parse_duration(test_case.text).map(|digits| { + [ + digits.years, + digits.months, + digits.days, + digits.hours, + digits.minutes, + digits.seconds, + ] + }); + + assert_eq!( + digits, + test_case + .expected_digits + .map(|parts| parts.map(str::to_owned)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_moment_when_shifting_by_duration_then_matches_python() { + let test_cases = [ + ShiftTestCase { + description: "month end clamps to a leap February", + moment: Moment::Timestamp { + local: local(civil(2024, 1, 31), (12, 0, 0, 0)), + fold: true, + }, + span: DurationSpan { + months: 1, + ..NO_SPAN + }, + direction: 1, + expected_moment: Ok(Moment::Timestamp { + local: local(civil(2024, 2, 29), (12, 0, 0, 0)), + fold: false, + }), + }, + ShiftTestCase { + description: "a date only moves by whole days", + moment: Moment::Date(civil(2024, 3, 1)), + span: DurationSpan { + hours: 36, + ..NO_SPAN + }, + direction: -1, + expected_moment: Ok(Moment::Date(civil(2024, 2, 29))), + }, + ShiftTestCase { + description: "a timedelta beyond its day limit", + moment: Moment::Date(civil(2024, 3, 1)), + span: DurationSpan { + days: 1_000_000_000, + ..NO_SPAN + }, + direction: 1, + expected_moment: Err(CursorError::Overflow( + "days=1000000000; must have magnitude <= 999999999".to_owned(), + )), + }, + ]; + + for test_case in test_cases { + assert_eq!( + shift_by_duration(test_case.moment, &test_case.span, test_case.direction), + test_case.expected_moment, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cursor/src/algebra/tests/test_intervals.rs b/crates/sqlbuild-cursor/src/algebra/tests/test_intervals.rs new file mode 100644 index 0000000000..76b9926f0b --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/tests/test_intervals.rs @@ -0,0 +1,71 @@ +use crate::algebra::errors::CursorError; +use crate::algebra::main::render::render; +use crate::algebra::main::split_into_batches::split_into_batches; +use crate::algebra::models::{Grain, Interval}; +use crate::algebra::tests::helpers::{date, integer, naive}; +use crate::algebra::tests::test_types::{NoZones, SplitTestCase}; + +#[test] +fn given_aligned_interval_when_splitting_then_batches_cover_it() { + let test_cases = [ + SplitTestCase { + description: "monthly batches of two across a leap February", + start: naive(2023, 12, 1, 0), + end: naive(2024, 5, 1, 0), + grain: Some(Grain::Month), + step: 2, + expected_starts: Ok(vec![ + "2023-12-01T00:00:00".to_owned(), + "2024-02-01T00:00:00".to_owned(), + "2024-04-01T00:00:00".to_owned(), + ]), + }, + SplitTestCase { + description: "a single short day batch", + start: date(2024, 2, 29), + end: date(2024, 3, 1), + grain: Some(Grain::Day), + step: 7, + expected_starts: Ok(vec!["2024-02-29".to_owned()]), + }, + SplitTestCase { + description: "integer batches", + start: integer(0), + end: integer(5), + grain: None, + step: 2, + expected_starts: Ok(vec!["0".to_owned(), "2".to_owned(), "4".to_owned()]), + }, + SplitTestCase { + description: "a zero step is rejected", + start: integer(0), + end: integer(5), + grain: None, + step: 0, + expected_starts: Err(CursorError::Algebra( + "cursor batch step must be positive".to_owned(), + )), + }, + ]; + + for test_case in test_cases { + let interval = Interval { + start: test_case.start, + end: test_case.end, + grain: test_case.grain, + }; + + let starts = split_into_batches(&interval, test_case.step, &NoZones).map(|batches| { + batches + .iter() + .map(|batch| render(&batch.start, &NoZones).unwrap_or_default()) + .collect::>() + }); + + assert_eq!( + starts, test_case.expected_starts, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cursor/src/algebra/tests/test_scalars.rs b/crates/sqlbuild-cursor/src/algebra/tests/test_scalars.rs new file mode 100644 index 0000000000..465ca72319 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/tests/test_scalars.rs @@ -0,0 +1,151 @@ +use crate::algebra::errors::CursorError; +use crate::algebra::main::floor_to_grain::floor_to_grain; +use crate::algebra::main::next_boundary::next_boundary; +use crate::algebra::main::normalize_timestamp_text::normalize_timestamp_text; +use crate::algebra::main::render::render; +use crate::algebra::models::{Grain, ZoneOffset}; +use crate::algebra::tests::helpers::{civil, date, integer, local, naive, timestamp}; +use crate::algebra::tests::test_types::{NoZones, RenderTestCase, StepTestCase, TextTestCase}; + +const INDIA_OFFSET: i64 = 19_800_000_000; + +#[test] +fn given_cursor_value_when_flooring_and_stepping_then_matches_python_datetime() { + let test_cases = [ + StepTestCase { + description: "leap-day timestamp floored to the month steps to March", + value: timestamp( + local(civil(2024, 2, 29), (13, 45, 12, 345_678)), + ZoneOffset::Naive, + ), + grain: Grain::Month, + expected_floor: naive(2024, 2, 1, 0), + expected_next: Ok(naive(2024, 3, 1, 0)), + }, + StepTestCase { + description: "hour grain keeps a fixed offset and steps one hour", + value: timestamp( + local(civil(2024, 12, 31), (23, 30, 0, 0)), + ZoneOffset::Fixed(INDIA_OFFSET), + ), + grain: Grain::Hour, + expected_floor: timestamp( + local(civil(2024, 12, 31), (23, 0, 0, 0)), + ZoneOffset::Fixed(INDIA_OFFSET), + ), + expected_next: Ok(timestamp( + local(civil(2025, 1, 1), (0, 0, 0, 0)), + ZoneOffset::Fixed(INDIA_OFFSET), + )), + }, + StepTestCase { + description: "a day-grained date is already aligned and steps a day", + value: date(2023, 2, 28), + grain: Grain::Day, + expected_floor: date(2023, 2, 28), + expected_next: Ok(date(2023, 3, 1)), + }, + StepTestCase { + description: "the last year fails as date.replace does", + value: date(9999, 7, 4), + grain: Grain::Year, + expected_floor: date(9999, 1, 1), + expected_next: Err(CursorError::Value("year 10000 is out of range".to_owned())), + }, + StepTestCase { + description: "the last second fails as datetime addition does", + value: timestamp( + local(civil(9999, 12, 31), (23, 59, 59, 1)), + ZoneOffset::Naive, + ), + grain: Grain::Second, + expected_floor: timestamp( + local(civil(9999, 12, 31), (23, 59, 59, 0)), + ZoneOffset::Naive, + ), + expected_next: Err(CursorError::Overflow("date value out of range".to_owned())), + }, + StepTestCase { + description: "integers are their own partitions", + value: integer(41), + grain: Grain::Day, + expected_floor: integer(41), + expected_next: Ok(integer(42)), + }, + ]; + + for test_case in test_cases { + let floored = floor_to_grain(&test_case.value, Some(test_case.grain)); + let next = next_boundary(&floored, Some(test_case.grain)); + + assert_eq!( + floored, test_case.expected_floor, + "{}", + test_case.description + ); + assert_eq!(next, test_case.expected_next, "{}", test_case.description); + } +} + +#[test] +fn given_timestamp_text_when_normalizing_then_joins_the_zone_suffix() { + let test_cases = [ + TextTestCase { + description: "space before an offset with a separator", + text: " 2024-01-01 00:00:00 +05:30 ", + expected_text: "2024-01-01 00:00:00+05:30", + }, + TextTestCase { + description: "Unicode spaces and digits before a compact offset", + text: "2024-01-01 00:00:00\u{3000}-\u{0660}800", + expected_text: "2024-01-01 00:00:00-\u{0660}800", + }, + TextTestCase { + description: "a UTC designator", + text: "2024-01-01T10:00:00 \tZ", + expected_text: "2024-01-01T10:00:00Z", + }, + TextTestCase { + description: "an hour-only offset is left alone", + text: "2024-01-01 00:00:00 +05", + expected_text: "2024-01-01 00:00:00 +05", + }, + ]; + + for test_case in test_cases { + assert_eq!( + normalize_timestamp_text(test_case.text), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_cursor_value_when_rendering_then_matches_isoformat() { + let test_cases = [ + RenderTestCase { + description: "an early year with microseconds and a second-level offset", + value: timestamp( + local(civil(5, 1, 1), (0, 0, 0, 7)), + ZoneOffset::Fixed(-10_807_000_003), + ), + expected_text: Ok("0005-01-01T00:00:00.000007-03:00:07.000003".to_owned()), + }, + RenderTestCase { + description: "a date", + value: date(2024, 2, 29), + expected_text: Ok("2024-02-29".to_owned()), + }, + ]; + + for test_case in test_cases { + assert_eq!( + render(&test_case.value, &NoZones), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cursor/src/algebra/tests/test_types.rs b/crates/sqlbuild-cursor/src/algebra/tests/test_types.rs new file mode 100644 index 0000000000..0a3f3f2c7b --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/tests/test_types.rs @@ -0,0 +1,48 @@ +use crate::algebra::errors::CursorError; +use crate::algebra::models::{DurationSpan, Grain, Moment, Scalar}; + +/// Zone rules for values that never carry an external zone. +pub(crate) struct NoZones; + +pub(crate) struct StepTestCase { + pub(crate) description: &'static str, + pub(crate) value: Scalar, + pub(crate) grain: Grain, + pub(crate) expected_floor: Scalar, + pub(crate) expected_next: Result, +} + +pub(crate) struct SplitTestCase { + pub(crate) description: &'static str, + pub(crate) start: Scalar, + pub(crate) end: Scalar, + pub(crate) grain: Option, + pub(crate) step: i64, + pub(crate) expected_starts: Result, CursorError>, +} + +pub(crate) struct DurationTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) expected_digits: Option<[&'static str; 6]>, +} + +pub(crate) struct ShiftTestCase { + pub(crate) description: &'static str, + pub(crate) moment: Moment, + pub(crate) span: DurationSpan, + pub(crate) direction: i64, + pub(crate) expected_moment: Result, +} + +pub(crate) struct TextTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct RenderTestCase { + pub(crate) description: &'static str, + pub(crate) value: Scalar, + pub(crate) expected_text: Result, +} diff --git a/crates/sqlbuild-cursor/src/algebra/types.rs b/crates/sqlbuild-cursor/src/algebra/types.rs new file mode 100644 index 0000000000..a1629d1a41 --- /dev/null +++ b/crates/sqlbuild-cursor/src/algebra/types.rs @@ -0,0 +1,23 @@ +//! The time zone lookups an aware timestamp's caller supplies. + +use crate::algebra::errors::CursorError; +use crate::algebra::models::LocalDateTime; + +/// UTC conversion for zones whose offsets the algebra cannot compute itself. +pub trait ZoneRules { + /// `value.astimezone(UTC).replace(tzinfo=None)` for the wall time in the `template` zone. + fn to_utc( + &self, + template: usize, + local: &LocalDateTime, + fold: bool, + ) -> Result; + + /// `value.utcoffset()` in microseconds, or `None` when the zone reports no offset. + fn utc_offset( + &self, + template: usize, + local: &LocalDateTime, + fold: bool, + ) -> Result, CursorError>; +} diff --git a/crates/sqlbuild-cursor/src/lib.rs b/crates/sqlbuild-cursor/src/lib.rs new file mode 100644 index 0000000000..d5963ee3a5 --- /dev/null +++ b/crates/sqlbuild-cursor/src/lib.rs @@ -0,0 +1,5 @@ +//! Cursor values, grains, intervals and durations with Python `datetime` results and errors. + +#![forbid(unsafe_code)] + +pub mod algebra; diff --git a/crates/sqlbuild-pysource/src/lib.rs b/crates/sqlbuild-pysource/src/lib.rs index ff28e1e1da..2faf6d791c 100644 --- a/crates/sqlbuild-pysource/src/lib.rs +++ b/crates/sqlbuild-pysource/src/lib.rs @@ -7,5 +7,6 @@ pub mod definition_locations; pub mod literal_sql; pub mod macro_modules; pub mod module_parse; +pub mod rule_evidence; pub mod string_literals; pub mod udf_functions; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/cases.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/cases.rs new file mode 100644 index 0000000000..411e7b55b5 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/cases.rs @@ -0,0 +1,125 @@ +//! The `RuleCase(...)` calls a harness call's `test_case` argument statically resolves to. + +use std::collections::{HashMap, HashSet}; + +use ruff_python_ast::{Expr, ExprCall, Stmt}; + +use crate::rule_evidence::_helpers::imports::{ + is_imported_symbol, is_parametrize_call, parameter_names, +}; +use crate::rule_evidence::constants::{ + MIN_PARAMETRIZE_ARGUMENTS, PUBLIC_HARNESS_MODULES, RULE_CASE, +}; +use crate::rule_evidence::types::Imports; + +/// The module bindings case resolution reads. +pub(crate) struct CaseScope<'a, 'b> { + pub(crate) imports: &'b Imports, + pub(crate) assignments: &'b HashMap<&'a str, &'a Expr>, + pub(crate) constructors: &'b HashSet, +} + +fn harness_modules() -> Vec { + PUBLIC_HARNESS_MODULES + .iter() + .map(|module| (*module).to_owned()) + .collect() +} + +fn declared_cases<'a>( + expression: &'a Expr, + field: Option<&str>, + scope: &CaseScope<'a, '_>, + seen: &HashSet<&'a str>, +) -> Vec<&'a ExprCall> { + if let Expr::Name(name) = expression + && let Some(value) = scope.assignments.get(name.id.as_str()) + && !seen.contains(name.id.as_str()) + { + let mut seen: HashSet<&'a str> = seen.clone(); + seen.insert(name.id.as_str()); + return declared_cases(value, field, scope, &seen); + } + let items: Option<&'a [Expr]> = match expression { + Expr::List(list) => Some(&list.elts), + Expr::Tuple(tuple) => Some(&tuple.elts), + Expr::Set(set) => Some(&set.elts), + _ => None, + }; + if let Some(items) = items { + return items + .iter() + .flat_map(|item| declared_cases(item, field, scope, seen)) + .collect(); + } + let Expr::Call(call) = expression else { + return Vec::new(); + }; + let Some(field) = field else { + if is_imported_symbol(&call.func, scope.imports, &harness_modules(), RULE_CASE) { + return vec![call]; + } + return Vec::new(); + }; + let Expr::Name(constructor) = &*call.func else { + return Vec::new(); + }; + if !scope.constructors.contains(constructor.id.as_str()) { + return Vec::new(); + } + call.arguments + .keywords + .iter() + .find(|keyword| { + keyword + .arg + .as_ref() + .is_some_and(|arg| arg.as_str() == field) + }) + .map(|keyword| declared_cases(&keyword.value, None, scope, seen)) + .unwrap_or_default() +} + +/// The values `pytest.mark.parametrize` binds to `parameter` on `owner`. +fn parametrized_values<'a>( + decorators: &'a [ruff_python_ast::Decorator], + parameter: &str, + imports: &Imports, +) -> Option<&'a Expr> { + decorators.iter().find_map(|decorator| { + let Expr::Call(call) = &decorator.expression else { + return None; + }; + let args: &[Expr] = &call.arguments.args; + (is_parametrize_call(&call.func, imports) + && args.len() >= MIN_PARAMETRIZE_ARGUMENTS + && parameter_names(&args[0]) == [parameter]) + .then(|| &args[1]) + }) +} + +/// The case declarations a `test_case` argument names, directly or through `parametrize`. +pub(crate) fn case_declarations<'a>( + expression: &'a Expr, + owner: &'a Stmt, + scope: &CaseScope<'a, '_>, +) -> Vec<&'a ExprCall> { + let direct: Vec<&'a ExprCall> = declared_cases(expression, None, scope, &HashSet::new()); + if !direct.is_empty() { + return direct; + } + let (parameter, field): (&str, Option<&str>) = match expression { + Expr::Name(name) => (name.id.as_str(), None), + Expr::Attribute(attribute) => match &*attribute.value { + Expr::Name(name) => (name.id.as_str(), Some(attribute.attr.as_str())), + _ => return Vec::new(), + }, + _ => return Vec::new(), + }; + let Stmt::FunctionDef(function) = owner else { + return Vec::new(); + }; + parametrized_values(&function.decorator_list, parameter, scope.imports) + .map(|values| declared_cases(values, field, scope, &HashSet::new())) + .unwrap_or_default() +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/evidence.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/evidence.rs new file mode 100644 index 0000000000..91a50c9441 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/evidence.rs @@ -0,0 +1,148 @@ +//! Test files under `tests/` and the harness cases each one statically declares for a rule. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +use ruff_python_ast::{Expr, ExprCall, Stmt}; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::models::{CpythonNode, ParsedSource, SourceLines}; +use crate::rule_evidence::_helpers::cases::{CaseScope, case_declarations}; +use crate::rule_evidence::_helpers::imports::{ + assignments, constructors, imports, is_imported_symbol, keyword, read_source, +}; +use crate::rule_evidence::_helpers::owners::{ + is_statically_dead, is_statically_skipped, test_owner, +}; +use crate::rule_evidence::_helpers::tree::NodeTree; +use crate::rule_evidence::constants::{ + EVALUATE_RULE, PUBLIC_HARNESS_MODULES, PYTHON_SUFFIX, RULE_KEYWORD, TEST_CASE_KEYWORD, + TEST_FILE_PREFIX, TEST_FILE_SUFFIX, +}; +use crate::rule_evidence::models::{RuleTarget, RuleTestEvidence}; +use crate::rule_evidence::types::Imports; + +/// Every `*.py` path below `directory`, without following directory links. +pub(crate) fn python_files(directory: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(directory) else { + return Vec::new(); + }; + let mut found: Vec = Vec::new(); + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + if entry.file_type().is_ok_and(|kind| kind.is_dir()) { + found.extend(python_files(&path)); + } + if path + .extension() + .is_some_and(|suffix| suffix == PYTHON_SUFFIX) + { + found.push(path); + } + } + found +} + +/// Whether pytest collects the file at `path` by its name. +pub(crate) fn is_test_file(path: &Path) -> bool { + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + name.starts_with(TEST_FILE_PREFIX) || name.ends_with(TEST_FILE_SUFFIX) +} + +/// The module facts every harness call in one test file is read against. +struct FileScope<'a, 'b> { + tree: NodeTree<'a>, + cases: CaseScope<'a, 'b>, + harness: Vec, + lines: &'b SourceLines, +} + +/// The cases one harness call declares, if it is a live test's call naming the rule. +fn call_evidence( + scope: &FileScope<'_, '_>, + (index, call): (usize, &ExprCall), + (relative_path, target): (&str, &RuleTarget<'_>), +) -> Vec { + let tree: &NodeTree<'_> = &scope.tree; + let module_imports: &Imports = scope.cases.imports; + let Some(owner) = test_owner(tree, index) else { + return Vec::new(); + }; + if is_statically_dead(tree, index, owner) || is_statically_skipped(tree, owner, module_imports) + { + return Vec::new(); + } + let (Some(rule), Some(case)) = ( + keyword(call, RULE_KEYWORD), + keyword(call, TEST_CASE_KEYWORD), + ) else { + return Vec::new(); + }; + let Some(owner_statement @ Stmt::FunctionDef(function)) = tree.stmt(owner) else { + return Vec::new(); + }; + if !is_imported_symbol(rule, module_imports, &target.modules, target.check_name) { + return Vec::new(); + } + let owner_name: String = format!("{relative_path}:{}", function.name.as_str()); + case_declarations(case, owner_statement, &scope.cases) + .into_iter() + .map(|declared| { + let (line, column) = + source_position(scope.lines, ruff_text_size::Ranged::start(declared)); + RuleTestEvidence { + path: relative_path.to_owned(), + line, + column: column + 1, + owner: owner_name.clone(), + } + }) + .collect() +} + +/// The harness cases the test file at `path` declares for the rule. +pub(crate) fn file_evidence( + path: &Path, + (relative_path, target): (&str, &RuleTarget<'_>), + python_version: (u8, u8), +) -> Vec { + let Some(parsed): Option = + parse_cpython_module(&read_source(path), python_version) + else { + return Vec::new(); + }; + let body: &[Stmt] = &parsed.module.body; + let module_imports: Imports = imports(body); + let module_assignments: HashMap<&str, &Expr> = assignments(body); + let module_constructors: HashSet = constructors(body, &module_imports); + let scope = FileScope { + tree: NodeTree::new(body), + cases: CaseScope { + imports: &module_imports, + assignments: &module_assignments, + constructors: &module_constructors, + }, + harness: PUBLIC_HARNESS_MODULES + .iter() + .map(|module| (*module).to_owned()) + .collect(), + lines: &parsed.lines, + }; + let mut found: Vec = Vec::new(); + for (index, node) in scope.tree.nodes.iter().enumerate() { + if let CpythonNode::Expr(Expr::Call(call)) = node + && is_imported_symbol(&call.func, &module_imports, &scope.harness, EVALUATE_RULE) + { + found.extend(call_evidence( + &scope, + (index, call), + (relative_path, target), + )); + } + } + found +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/imports.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/imports.rs new file mode 100644 index 0000000000..7102eee673 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/imports.rs @@ -0,0 +1,207 @@ +//! Import bindings, module assignments and the pytest and harness names a test module uses. + +use std::collections::{HashMap, HashSet}; + +use ruff_python_ast::{Expr, ExprCall, Number, Stmt}; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::rule_evidence::constants::{ + FLOAT_ZERO, PARAMETRIZE_ATTRIBUTE, PYTEST_MARK_ATTRIBUTE, PYTEST_MODULE_NAME, +}; +use crate::rule_evidence::types::Imports; + +/// Each top-level import's bound name; relative `from` imports bind nothing here. +pub(crate) fn imports(body: &[Stmt]) -> Imports { + let mut result: Imports = HashMap::new(); + for statement in body { + match statement { + Stmt::Import(import) => { + for alias in &import.names { + let name: &str = alias.name.as_str(); + let bound: &str = alias.asname.as_ref().map_or_else( + || name.split('.').next().unwrap_or(name), + |asname| asname.as_str(), + ); + result.insert(bound.to_owned(), (name.to_owned(), None)); + } + } + Stmt::ImportFrom(import) if import.level == 0 => { + let Some(module) = &import.module else { + continue; + }; + for alias in &import.names { + let bound: &str = alias.asname.as_ref().unwrap_or(&alias.name).as_str(); + result.insert( + bound.to_owned(), + ( + module.as_str().to_owned(), + Some(alias.name.as_str().to_owned()), + ), + ); + } + } + _ => {} + } + } + result +} + +/// Each module-level name's last assigned value. +pub(crate) fn assignments(body: &[Stmt]) -> HashMap<&str, &Expr> { + let mut result: HashMap<&str, &Expr> = HashMap::new(); + for statement in body { + let (targets, value): (Vec<&Expr>, Option<&Expr>) = match statement { + Stmt::Assign(assign) => (assign.targets.iter().collect(), Some(&assign.value)), + Stmt::AnnAssign(assign) => (vec![&*assign.target], assign.value.as_deref()), + _ => continue, + }; + let Some(value) = value else { + continue; + }; + for target in targets { + if let Expr::Name(name) = target { + result.insert(name.id.as_str(), value); + } + } + } + result +} + +/// The module's classes and every name imported from a module. +pub(crate) fn constructors(body: &[Stmt], imports: &Imports) -> HashSet { + let mut declared: HashSet = body + .iter() + .filter_map(|statement| match statement { + Stmt::ClassDef(class) => Some(class.name.as_str().to_owned()), + _ => None, + }) + .collect(); + declared.extend( + imports + .iter() + .filter(|(_, (_, symbol))| symbol.is_some()) + .map(|(name, _)| name.clone()), + ); + declared +} + +/// Whether `expression` names `symbol` imported from one of `modules`. +pub(crate) fn is_imported_symbol( + expression: &Expr, + imports: &Imports, + modules: &[String], + symbol: &str, +) -> bool { + match expression { + Expr::Name(name) => imports + .get(name.id.as_str()) + .is_some_and(|(module, imported)| { + modules.contains(module) && imported.as_deref() == Some(symbol) + }), + Expr::Attribute(attribute) => { + let Expr::Name(base) = &*attribute.value else { + return false; + }; + let Some((module, imported)) = imports.get(base.id.as_str()) else { + return false; + }; + if attribute.attr.as_str() != symbol { + return false; + } + let qualified: String = match imported { + None => module.clone(), + Some(imported) => format!("{module}.{imported}"), + }; + modules.contains(&qualified) + } + _ => false, + } +} + +/// The value of the first `name=` keyword of `call`. +pub(crate) fn keyword<'a>(call: &'a ExprCall, name: &str) -> Option<&'a Expr> { + call.arguments + .keywords + .iter() + .find(|keyword| keyword.arg.as_ref().is_some_and(|arg| arg.as_str() == name)) + .map(|keyword| &keyword.value) +} + +/// Whether `expression` is `pytest.mark.` through a plain `import pytest`. +pub(crate) fn is_pytest_mark(expression: &Expr, imports: &Imports, mark: &str) -> bool { + let Expr::Attribute(attribute) = expression else { + return false; + }; + let Expr::Attribute(marks) = &*attribute.value else { + return false; + }; + let Expr::Name(module) = &*marks.value else { + return false; + }; + attribute.attr.as_str() == mark + && marks.attr.as_str() == PYTEST_MARK_ATTRIBUTE + && imports.get(module.id.as_str()) == Some(&(PYTEST_MODULE_NAME.to_owned(), None)) +} + +pub(crate) fn is_parametrize_call(expression: &Expr, imports: &Imports) -> bool { + is_pytest_mark(expression, imports, PARAMETRIZE_ATTRIBUTE) +} + +/// The parameter names a `parametrize` argument-names expression declares. +pub(crate) fn parameter_names(expression: &Expr) -> Vec { + match expression { + Expr::StringLiteral(text) => text + .value + .to_str() + .split(',') + .map(|name| python_strip(name).to_owned()) + .collect(), + Expr::List(list) => string_items(&list.elts), + Expr::Tuple(tuple) => string_items(&tuple.elts), + _ => Vec::new(), + } +} + +fn string_items(items: &[Expr]) -> Vec { + let mut values: Vec = Vec::with_capacity(items.len()); + for item in items { + let Expr::StringLiteral(text) = item else { + return Vec::new(); + }; + values.push(text.value.to_str().to_owned()); + } + values +} + +/// `bool(node.value)` for an `ast.Constant`, or `None` for any other expression. +pub(crate) fn constant_truth(expression: &Expr) -> Option { + Some(match expression { + Expr::BooleanLiteral(literal) => literal.value, + Expr::NoneLiteral(_) => false, + Expr::EllipsisLiteral(_) => true, + Expr::StringLiteral(text) => !text.value.is_empty(), + Expr::BytesLiteral(bytes) => bytes.value.bytes().next().is_some(), + Expr::NumberLiteral(number) => match &number.value { + Number::Int(value) => *value != ruff_python_ast::Int::ZERO, + Number::Float(value) => *value != FLOAT_ZERO, + Number::Complex { real, imag } => *real != FLOAT_ZERO || *imag != FLOAT_ZERO, + }, + _ => return None, + }) +} + +/// Whether `expression` is the `True` constant. +pub(crate) fn is_true_constant(expression: &Expr) -> bool { + matches!(expression, Expr::BooleanLiteral(literal) if literal.value) +} + +/// The UTF-8 text of the file at `path`; an unreadable file reads as empty, as Python skips it. +pub(crate) fn read_source(path: &std::path::Path) -> String { + let Ok(bytes) = std::fs::read(path) else { + return String::new(); + }; + let Ok(text) = String::from_utf8(bytes) else { + return String::new(); + }; + text +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs new file mode 100644 index 0000000000..af1a7d23cc --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod cases; +pub(crate) mod evidence; +pub(crate) mod imports; +pub(crate) mod owners; +pub(crate) mod paths; +pub(crate) mod tree; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/owners.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/owners.rs new file mode 100644 index 0000000000..16f3201da4 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/owners.rs @@ -0,0 +1,127 @@ +//! The test function that owns a harness call, and whether that call can ever run. + +use ruff_python_ast::{Decorator, Expr, Stmt, StmtFunctionDef}; + +use crate::cpython_tree::models::CpythonNode; +use crate::rule_evidence::_helpers::imports::{ + constant_truth, is_pytest_mark, is_true_constant, keyword, +}; +use crate::rule_evidence::_helpers::tree::NodeTree; +use crate::rule_evidence::constants::{ + CONDITION_KEYWORD, SKIP_MARK, SKIPIF_MARK, TEST_CLASS_PREFIX, TEST_FUNCTION_PREFIX, +}; +use crate::rule_evidence::types::Imports; + +fn class_index(tree: &NodeTree<'_>, index: Option) -> Option { + let index: usize = index?; + matches!(tree.stmt(index), Some(Stmt::ClassDef(_))).then_some(index) +} + +/// The top-level or `Test*`-class `test_*` function enclosing `node`, unless a lambda or class intervenes. +pub(crate) fn test_owner(tree: &NodeTree<'_>, node: usize) -> Option { + let mut current: usize = node; + while let Some(parent) = tree.parents[current] { + current = parent; + if matches!(tree.nodes[current], CpythonNode::Expr(Expr::Lambda(_))) + || class_index(tree, Some(current)).is_some() + { + return None; + } + let Some(Stmt::FunctionDef(function)) = tree.stmt(current) else { + continue; + }; + if !function.name.as_str().starts_with(TEST_FUNCTION_PREFIX) { + return None; + } + let Some(parent) = tree.parents[current] else { + return Some(current); + }; + let Some(Stmt::ClassDef(class)) = + class_index(tree, Some(parent)).and_then(|class| tree.stmt(class)) + else { + return None; + }; + return (class.name.as_str().starts_with(TEST_CLASS_PREFIX) + && tree.parents[parent].is_none()) + .then_some(current); + } + None +} + +/// Whether one decorator is `pytest.mark.skip` or an always-true `pytest.mark.skipif`. +fn skips_statically(decorator: &Decorator, imports: &Imports) -> bool { + let call = match &decorator.expression { + Expr::Call(call) => Some(call), + _ => None, + }; + let target: &Expr = call.map_or(&decorator.expression, |call| &call.func); + if is_pytest_mark(target, imports, SKIP_MARK) { + return true; + } + let Some(call) = call else { + return false; + }; + if !is_pytest_mark(&call.func, imports, SKIPIF_MARK) { + return false; + } + let condition: Option<&Expr> = match call.arguments.args.first() { + Some(first) => Some(first), + None => keyword(call, CONDITION_KEYWORD), + }; + condition.is_some_and(is_true_constant) +} + +fn skips(decorators: &[Decorator], imports: &Imports) -> bool { + decorators + .iter() + .any(|decorator| skips_statically(decorator, imports)) +} + +/// Whether the owner or its enclosing class carries `skip` or an always-true `skipif`. +pub(crate) fn is_statically_skipped(tree: &NodeTree<'_>, owner: usize, imports: &Imports) -> bool { + let Some(Stmt::FunctionDef(function)) = tree.stmt(owner) else { + return false; + }; + let function: &StmtFunctionDef = function; + if skips(&function.decorator_list, imports) { + return true; + } + match class_index(tree, tree.parents[owner]).and_then(|class| tree.stmt(class)) { + Some(Stmt::ClassDef(class)) => skips(&class.decorator_list, imports), + _ => false, + } +} + +/// The test and body of an `If`, an `elif` or a `While` node, when the test is constant. +fn constant_branch<'a>(node: &CpythonNode<'a>) -> Option<(bool, &'a [Stmt])> { + let (test, body): (&Expr, &[Stmt]) = match node { + CpythonNode::Stmt(Stmt::If(statement)) => (&statement.test, &statement.body), + CpythonNode::Stmt(Stmt::While(statement)) => (&statement.test, &statement.body), + CpythonNode::Elif(clauses) => { + let clause = clauses.first()?; + (clause.test.as_ref()?, &clause.body) + } + _ => return None, + }; + Some((constant_truth(test)?, body)) +} + +/// Whether a constant `if` or `while` test keeps `node` from ever running inside `owner`. +pub(crate) fn is_statically_dead(tree: &NodeTree<'_>, node: usize, owner: usize) -> bool { + let mut current: usize = node; + while current != owner { + let Some(parent) = tree.parents[current] else { + return true; + }; + if let Some((truth, body)) = constant_branch(&tree.nodes[parent]) { + let in_body: bool = tree + .stmt(current) + .is_some_and(|statement| body.iter().any(|item| std::ptr::eq(item, statement))); + if in_body != truth { + return true; + } + } + current = parent; + } + false +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs new file mode 100644 index 0000000000..2b87ef6f22 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs @@ -0,0 +1,173 @@ +//! Filesystem paths resolved, compared and searched the way `pathlib` does for rule modules. + +use std::collections::BTreeSet; +use std::path::{Component, Path, PathBuf}; + +use ruff_python_ast::Stmt; + +use crate::cpython_tree::models::CpythonNode; +use crate::rule_evidence::constants::{ + INIT_FILE_NAME, INIT_MODULE_NAME, PYTHON_SUFFIX, RULES_DIRECTORY_NAME, STAR_IMPORT_NAME, +}; + +/// `Path.resolve()`: symlinks resolved through the existing prefix, the rest kept lexically. +pub(crate) fn resolve(path: &Path) -> PathBuf { + if let Ok(resolved) = std::fs::canonicalize(path) { + return resolved; + } + let Some(parent) = path.parent().filter(|parent| *parent != path) else { + return path.to_path_buf(); + }; + let base: PathBuf = resolve(parent); + match path.components().next_back() { + Some(Component::ParentDir) => base.parent().map_or(base.clone(), Path::to_path_buf), + Some(Component::CurDir) | None => base, + Some(component) => base.join(component.as_os_str()), + } +} + +fn is_file(path: &Path) -> bool { + path.is_file() +} + +/// `base.joinpath(*parts)`'s module file, if it resolves under the rules root. +fn module_file(base: &Path, parts: &[String], rules_root: &Path) -> Option { + let candidate: PathBuf = parts + .iter() + .fold(base.to_path_buf(), |path, part| path.join(part)); + let package: PathBuf = candidate.join(INIT_FILE_NAME); + let module: Option = + (!parts.is_empty()).then(|| candidate.with_extension(PYTHON_SUFFIX)); + for path in std::iter::once(package).chain(module) { + if is_file(&path) { + let resolved: PathBuf = resolve(&path); + return resolved.starts_with(rules_root).then_some(resolved); + } + } + None +} + +/// Package initializers executed before `path` under the rules root. +pub(crate) fn package_inits(path: &Path, rules_root: &Path) -> Vec { + let mut inits: Vec = Vec::new(); + for parent in path.ancestors().skip(1) { + if !parent.starts_with(rules_root) { + break; + } + let init: PathBuf = parent.join(INIT_FILE_NAME); + if !is_file(&init) { + continue; + } + let resolved: PathBuf = resolve(&init); + if resolved != path && resolved.starts_with(rules_root) { + inits.push(resolved); + } + } + inits +} + +/// Every repository-owned rules module one import statement can execute. +pub(crate) fn local_import_paths( + node: &CpythonNode<'_>, + source_path: &Path, + rules_root: &Path, +) -> Vec { + let project: PathBuf = rules_root + .parent() + .map_or(PathBuf::new(), Path::to_path_buf); + let mut targets: Vec<(PathBuf, Vec)> = Vec::new(); + match node { + CpythonNode::Stmt(Stmt::Import(import)) => { + for alias in &import.names { + let parts: Vec = + alias.name.as_str().split('.').map(str::to_owned).collect(); + if parts[0] == RULES_DIRECTORY_NAME { + targets.push((project.clone(), parts)); + } + } + } + CpythonNode::Stmt(Stmt::ImportFrom(import)) => { + let module_parts: Vec = + import.module.as_ref().map_or_else(Vec::new, |module| { + module.as_str().split('.').map(str::to_owned).collect() + }); + let mut base: PathBuf = project.clone(); + if import.level > 0 { + base = source_path + .parent() + .map_or(PathBuf::new(), Path::to_path_buf); + for _ in 1..import.level { + base = base.parent().map_or(base.clone(), Path::to_path_buf); + } + } else if module_parts.first().map(String::as_str) != Some(RULES_DIRECTORY_NAME) { + return Vec::new(); + } + targets.push((base.clone(), module_parts.clone())); + for alias in &import.names { + if alias.name.as_str() != STAR_IMPORT_NAME { + let mut parts: Vec = module_parts.clone(); + parts.push(alias.name.as_str().to_owned()); + targets.push((base.clone(), parts)); + } + } + } + _ => {} + } + let mut resolved: Vec = Vec::new(); + for (base, parts) in targets { + if let Some(module) = module_file(&base, &parts, rules_root) { + resolved.extend(package_inits(&module, rules_root)); + resolved.push(module); + } + } + resolved +} + +/// Whether each package from `root` down through `packages` has an `__init__.py`. +fn every_package_initialized(root: &Path, packages: &[String]) -> bool { + let mut package: PathBuf = root.to_path_buf(); + for name in packages { + package.push(name); + if !package.join(INIT_FILE_NAME).is_file() { + return false; + } + } + true +} + +/// The dotted names a rule module at `path` can be imported as from the project. +pub(crate) fn module_names( + path: &Path, + project_dir: &Path, + root: &Path, +) -> Option> { + let Ok(relative) = path.strip_prefix(root) else { + return None; + }; + let mut parts: Vec = relative + .components() + .map(|component| component.as_os_str().to_string_lossy().into_owned()) + .collect(); + if let Some(last) = parts.last_mut() + && let Some(stem) = Path::new(last.as_str()).file_stem() + && Path::new(last.as_str()).extension().is_some() + { + *last = stem.to_string_lossy().into_owned(); + } + if parts.last().map(String::as_str) == Some(INIT_MODULE_NAME) { + parts.pop(); + } + let mut names: BTreeSet = BTreeSet::from([parts.join(".")]); + for index in 0..parts.len() { + let package_parts: &[String] = &parts[index..parts.len() - 1]; + let package_root: PathBuf = parts[..index] + .iter() + .fold(project_dir.to_path_buf(), |path, part| path.join(part)); + let packaged: bool = + !package_parts.is_empty() && every_package_initialized(&package_root, package_parts); + if packaged { + names.insert(parts[index..].join(".")); + } + } + Some(names) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/tree.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/tree.rs new file mode 100644 index 0000000000..73961062d6 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/tree.rs @@ -0,0 +1,34 @@ +//! A parsed module's nodes in `ast.walk` order, each with the index of its parent node. + +use ruff_python_ast::Stmt; + +use crate::cpython_tree::main::child_nodes::child_nodes; +use crate::cpython_tree::models::CpythonNode; + +/// Every node below the module, breadth first; a top-level statement has no parent. +pub(crate) struct NodeTree<'a> { + pub(crate) nodes: Vec>, + pub(crate) parents: Vec>, +} + +impl<'a> NodeTree<'a> { + pub(crate) fn new(body: &'a [Stmt]) -> Self { + let mut nodes: Vec> = body.iter().map(CpythonNode::Stmt).collect(); + let mut parents: Vec> = vec![None; nodes.len()]; + let mut index: usize = 0; + while index < nodes.len() { + let children: Vec> = child_nodes(&nodes[index]); + parents.extend(std::iter::repeat_n(Some(index), children.len())); + nodes.extend(children); + index += 1; + } + Self { nodes, parents } + } + + pub(crate) fn stmt(&self, index: usize) -> Option<&'a Stmt> { + match self.nodes[index] { + CpythonNode::Stmt(statement) => Some(statement), + _ => None, + } + } +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/constants.rs b/crates/sqlbuild-pysource/src/rule_evidence/constants.rs new file mode 100644 index 0000000000..831fdd337a --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/constants.rs @@ -0,0 +1,25 @@ +//! Module names, file names and pytest attributes custom-rule evidence reads. + +pub(crate) const PUBLIC_HARNESS_MODULES: [&str; 2] = ["sqlbuild.rules", "sqlbuild.rules.testing"]; +pub(crate) const EVALUATE_RULE: &str = "evaluate_rule"; +pub(crate) const RULE_CASE: &str = "RuleCase"; +pub(crate) const RULE_KEYWORD: &str = "rule"; +pub(crate) const TEST_CASE_KEYWORD: &str = "test_case"; +pub(crate) const CONDITION_KEYWORD: &str = "condition"; +pub(crate) const INIT_MODULE_NAME: &str = "__init__"; +pub(crate) const INIT_FILE_NAME: &str = "__init__.py"; +pub(crate) const RULES_DIRECTORY_NAME: &str = "rules"; +pub(crate) const TESTS_DIRECTORY_NAME: &str = "tests"; +pub(crate) const PYTHON_SUFFIX: &str = "py"; +pub(crate) const STAR_IMPORT_NAME: &str = "*"; +pub(crate) const MIN_PARAMETRIZE_ARGUMENTS: usize = 2; +pub(crate) const PARAMETRIZE_ATTRIBUTE: &str = "parametrize"; +pub(crate) const PYTEST_MARK_ATTRIBUTE: &str = "mark"; +pub(crate) const PYTEST_MODULE_NAME: &str = "pytest"; +pub(crate) const SKIP_MARK: &str = "skip"; +pub(crate) const SKIPIF_MARK: &str = "skipif"; +pub(crate) const TEST_FUNCTION_PREFIX: &str = "test_"; +pub(crate) const TEST_FILE_PREFIX: &str = "test_"; +pub(crate) const TEST_FILE_SUFFIX: &str = "_test.py"; +pub(crate) const TEST_CLASS_PREFIX: &str = "Test"; +pub(crate) const FLOAT_ZERO: f64 = 0.0_f64; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/implementation_fingerprint.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/implementation_fingerprint.rs new file mode 100644 index 0000000000..7f15c6cd30 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/implementation_fingerprint.rs @@ -0,0 +1,21 @@ +//! `implementation_fingerprint`: SHA-256 of a rule file and its imported helper closure. + +use std::path::Path; + +use sha2::{Digest, Sha256}; + +/// The digest of each file's root-relative label and bytes, rule source first. +pub fn implementation_fingerprint( + source_path: &Path, + closure: &[impl AsRef], + root: &Path, +) -> std::io::Result { + let mut digest = Sha256::new(); + for path in std::iter::once(source_path).chain(closure.iter().map(AsRef::as_ref)) { + let label: &Path = path.strip_prefix(root).unwrap_or(path); + digest.update(label.to_string_lossy().as_bytes()); + digest.update(b"\0"); + digest.update(std::fs::read(path)?); + } + Ok(format!("{:x}", digest.finalize())) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs new file mode 100644 index 0000000000..f8c040ac98 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs @@ -0,0 +1,43 @@ +//! `import_closure`: the repository-owned helper modules one rule module imports. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::rule_evidence::_helpers::imports::read_source; +use crate::rule_evidence::_helpers::paths::{local_import_paths, package_inits, resolve}; +use crate::rule_evidence::constants::RULES_DIRECTORY_NAME; + +/// Every rules module and package initializer `source_path` executes, sorted; paths are resolved. +pub fn import_closure( + source_path: &Path, + project_dir: &Path, + python_version: (u8, u8), +) -> Vec { + let rules_root: PathBuf = resolve(&project_dir.join(RULES_DIRECTORY_NAME)); + let mut pending: Vec = vec![source_path.to_path_buf()]; + let mut visited: BTreeSet = BTreeSet::from([source_path.to_path_buf()]); + let mut helpers: BTreeSet = BTreeSet::new(); + while let Some(current) = pending.pop() { + let mut candidates: BTreeSet = + package_inits(¤t, &rules_root).into_iter().collect(); + let parsed: Option = + parse_cpython_module(&read_source(¤t), python_version); + if let Some(parsed) = &parsed { + let roots: Vec> = + parsed.module.body.iter().map(CpythonNode::Stmt).collect(); + for node in walk_breadth_first(roots) { + candidates.extend(local_import_paths(&node, ¤t, &rules_root)); + } + } + for candidate in candidates { + if visited.insert(candidate.clone()) { + helpers.insert(candidate.clone()); + pending.push(candidate); + } + } + } + helpers.into_iter().collect() +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs new file mode 100644 index 0000000000..e8b3ce047f --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs @@ -0,0 +1,3 @@ +pub mod implementation_fingerprint; +pub mod import_closure; +pub mod test_evidence; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/test_evidence.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/test_evidence.rs new file mode 100644 index 0000000000..f03a3b953e --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/test_evidence.rs @@ -0,0 +1,36 @@ +//! `test_evidence`: the public-harness cases that statically exercise one custom rule. + +use std::collections::BTreeSet; +use std::path::Path; + +use crate::rule_evidence::_helpers::evidence::{file_evidence, is_test_file, python_files}; +use crate::rule_evidence::_helpers::paths::module_names; +use crate::rule_evidence::constants::TESTS_DIRECTORY_NAME; +use crate::rule_evidence::models::{RuleTarget, RuleTestEvidence}; + +/// Sorted harness cases under `tests/` naming the rule; `None` when its source is outside `root`. +pub fn test_evidence( + (source_path, check_name): (&Path, &str), + (project_dir, root): (&Path, &Path), + python_version: (u8, u8), +) -> Option> { + let target = RuleTarget { + modules: module_names(source_path, project_dir, root)? + .into_iter() + .collect(), + check_name, + }; + let mut evidence: BTreeSet = BTreeSet::new(); + for path in python_files(&project_dir.join(TESTS_DIRECTORY_NAME)) { + if !is_test_file(&path) { + continue; + } + let relative: String = path + .strip_prefix(project_dir) + .unwrap_or(&path) + .to_string_lossy() + .into_owned(); + evidence.extend(file_evidence(&path, (&relative, &target), python_version)); + } + Some(evidence.into_iter().collect()) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/mod.rs b/crates/sqlbuild-pysource/src/rule_evidence/mod.rs new file mode 100644 index 0000000000..a3699a7187 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/mod.rs @@ -0,0 +1,10 @@ +//! Custom-rule source closures, implementation fingerprints and statically found test cases. + +pub(crate) mod _helpers; +mod constants; +pub mod main; +pub mod models; +pub(crate) mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/models.rs b/crates/sqlbuild-pysource/src/rule_evidence/models.rs new file mode 100644 index 0000000000..68acd03cdb --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/models.rs @@ -0,0 +1,17 @@ +//! One statically found custom-rule test case and the import bindings of one test module. + +/// One harness case: its file, CPython position and owning test. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RuleTestEvidence { + pub path: String, + pub line: usize, + pub column: usize, + pub owner: String, +} + +/// The rule a harness call must name: its importable module names and check function name. +#[derive(Debug)] +pub(crate) struct RuleTarget<'a> { + pub(crate) modules: Vec, + pub(crate) check_name: &'a str, +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests.rs new file mode 100644 index 0000000000..52b71770bf --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_rule_evidence.rs"] +mod rule_evidence; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs new file mode 100644 index 0000000000..b1599b4202 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs @@ -0,0 +1,31 @@ +use std::path::{Path, PathBuf}; + +const RULE_FILES: [(&str, &str); 4] = [ + ("rules/__init__.py", ""), + ( + "rules/custom.py", + "from rules.shared import util\nfrom .missing import nothing\n\ndef check(*, model, ctx):\n return []\n", + ), + ("rules/shared/__init__.py", ""), + ("rules/shared/util.py", "import os\n"), +]; + +/// A fresh project holding the rule files and `tests/test_rule.py` with `test_source`. +pub(crate) fn rule_project(name: &str, test_source: &str) -> PathBuf { + let root: PathBuf = std::env::temp_dir() + .join(format!("sqlbuild-rule-evidence-{}", std::process::id())) + .join(name); + let _ = std::fs::remove_dir_all(&root); + for (path, contents) in RULE_FILES + .into_iter() + .chain([("tests/test_rule.py", test_source)]) + { + write(&root.join(path), contents); + } + std::fs::canonicalize(&root).unwrap_or(root) +} + +fn write(path: &Path, contents: &str) { + let _ = std::fs::create_dir_all(path.parent().unwrap_or(path)); + let _ = std::fs::write(path, contents); +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs new file mode 100644 index 0000000000..5035eb0930 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs @@ -0,0 +1,76 @@ +use std::path::{Path, PathBuf}; + +use crate::rule_evidence::main::import_closure::import_closure; +use crate::rule_evidence::main::test_evidence::test_evidence; +use crate::rule_evidence::tests::helpers::rule_project; +use crate::rule_evidence::tests::test_types::EvidenceTestCase; + +const PYTHON: (u8, u8) = (3, 12); + +#[test] +fn given_rule_project_when_reading_evidence_then_matches_python_closure_and_cases() { + let test_cases = [ + EvidenceTestCase { + description: "parametrized cases, a dead branch and a skipped test", + project: "parametrized", + test_source: "import pytest\nfrom sqlbuild.rules.testing import RuleCase, evaluate_rule\nfrom rules.custom import check\n\nCASES = [RuleCase(description='a', source='', expected_finding_count=0), RuleCase(description='b', source='', expected_finding_count=0)]\n\n@pytest.mark.parametrize('test_case', CASES)\ndef test_cases(test_case):\n evaluate_rule(rule=check, test_case=test_case)\n if False:\n evaluate_rule(rule=check, test_case=RuleCase(description='dead', source='', expected_finding_count=0))\n\n@pytest.mark.skip\ndef test_skipped():\n evaluate_rule(rule=check, test_case=RuleCase(description='skip', source='', expected_finding_count=0))\n", + expected_closure: vec![ + "rules/__init__.py", + "rules/shared/__init__.py", + "rules/shared/util.py", + ], + expected_cases: vec![ + (5, 10, "tests/test_rule.py:test_cases"), + (5, 74, "tests/test_rule.py:test_cases"), + ], + }, + EvidenceTestCase { + description: "a harness call outside a test function counts nothing", + project: "module_level", + test_source: "from sqlbuild.rules.testing import RuleCase, evaluate_rule\nfrom rules.custom import check\n\nevaluate_rule(rule=check, test_case=RuleCase(description='a', source='', expected_finding_count=0))\n", + expected_closure: vec![ + "rules/__init__.py", + "rules/shared/__init__.py", + "rules/shared/util.py", + ], + expected_cases: vec![], + }, + ]; + + for test_case in test_cases { + let root: PathBuf = rule_project(test_case.project, test_case.test_source); + let source: PathBuf = root.join("rules/custom.py"); + let closure: Vec = import_closure(&source, &root, PYTHON); + let cases: Vec<(usize, usize, String)> = + test_evidence((&source, "check"), (&root, &root), PYTHON) + .unwrap_or_default() + .into_iter() + .map(|case| (case.line, case.column, case.owner)) + .collect(); + let relative: Vec<&Path> = closure + .iter() + .map(|path| path.strip_prefix(&root).unwrap_or(path)) + .collect(); + + assert_eq!( + relative, + test_case + .expected_closure + .iter() + .map(Path::new) + .collect::>(), + "{}", + test_case.description + ); + assert_eq!( + cases, + test_case + .expected_cases + .iter() + .map(|(line, column, owner)| (*line, *column, (*owner).to_owned())) + .collect::>(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs new file mode 100644 index 0000000000..6fc509556a --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(crate) struct EvidenceTestCase { + pub(crate) description: &'static str, + pub(crate) project: &'static str, + pub(crate) test_source: &'static str, + pub(crate) expected_closure: Vec<&'static str>, + pub(crate) expected_cases: Vec<(usize, usize, &'static str)>, +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/types.rs b/crates/sqlbuild-pysource/src/rule_evidence/types.rs new file mode 100644 index 0000000000..4b2935a62d --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/types.rs @@ -0,0 +1,6 @@ +//! The import bindings of one test module. + +use std::collections::HashMap; + +/// Each top-level imported name and its `(module, symbol)`; `import x` has no symbol. +pub(crate) type Imports = HashMap)>; diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index a434dcb14c..fc2f00f2b3 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -27,6 +27,7 @@ sqlbuild-cache.workspace = true sqlbuild-cli.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true +sqlbuild-cursor.workspace = true sqlbuild-discovery.workspace = true sqlbuild-model-config.workspace = true sqlbuild-pysource.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 6e8f6ebba4..7c99056e02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; -const WAREHOUSES_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouses"; +const WAREHOUSES_MODULE: &str = "sqlbuild._native"; const WAREHOUSE_SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const LAUNCH_RECORD_MODULE: &str = "sqlbuild.cli.entry.main._withhold_launch_record"; const DIFF_VALIDATION_MODULE: &str = "sqlbuild.cli.commands._helpers.diff.validation"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index b15b89bcfa..eed0b86fe3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -31,7 +31,7 @@ const DIAGNOSTICS_MODULE: &str = "sqlbuild.diagnostics.main.diagnostics_context" const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const CODED_ERROR_MODULE: &str = "sqlbuild.presentation.main.coded_error_text"; const TRANSIENT_LINE_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; -const SKILLS_MODULE: &str = "sqlbuild.cli.commands._helpers.skills.update"; +const SKILLS_MODULE: &str = "sqlbuild._native"; const DIFF_JSON_MODULE: &str = "sqlbuild.cli.commands._helpers.diff.json_output"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index bc33adc976..4f49628d00 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -1,5 +1,8 @@ pub(crate) mod argv; pub(crate) mod dispatch; pub(crate) mod entry; +pub(crate) mod progress; +pub(crate) mod progress_reporters; +pub(crate) mod progress_text; pub(crate) mod registration; pub(crate) mod surrogates; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs new file mode 100644 index 0000000000..50d1c0c5c7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs @@ -0,0 +1,352 @@ +//! The invocation-scoped native CLI projection of canonical lifecycle facts. + +use std::sync::{Mutex, MutexGuard}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyMapping, PyString}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::progress::main::claim_progress_terminal::claim_progress_terminal; +use sqlbuild_core::progress::main::close_progress_projection::close_progress_projection; +use sqlbuild_core::progress::main::consume_progress_event::consume_progress_event; +use sqlbuild_core::progress::main::progress_terminal_duration::progress_terminal_duration; +use sqlbuild_core::progress::models::{ + ProgressEvent, ProgressPayload, ProgressProjection, ProgressSettings, +}; +use sqlbuild_core::terminal_events::models::TerminalRequest; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; + +/// The module attribute holding the `ContextVar` of the installed projector. +const CONTEXT_ATTRIBUTE: &str = "_native_progress_projector_context"; +const CONTEXT_NAME: &str = "sqlbuild_native_progress_projector"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COORDINATOR_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const COORDINATOR_FUNCTION: &str = "shared_transient_line_coordinator"; +const CLAIM_OPTIONS: (&[&str], &[&str]) = ( + &["resource_name"], + &["resource_id", "resource_attempt_id", "attempt_number"], +); +const DEFAULT_INTENT: &str = "execute"; +const DEFAULT_STATEMENT_KIND: &str = "UNKNOWN"; + +/// Project canonical starts and correlate enriched terminal presentation. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeProgressProjector { + state: Mutex, + stream: Py, + coordinator: Py, +} + +fn payload_value<'py>(payload: &Bound<'py, PyAny>, key: &str) -> PyResult> { + payload.call_method1("get", (key,)) +} + +fn text_value(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return Ok(Some(value.extract()?)); + } + Ok(None) +} + +/// An `int` that is not a `bool`. +fn strict_integer(value: &Bound<'_, PyAny>) -> PyResult> { + if !value.is_instance_of::() || value.is_instance_of::() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +/// An `int` that is not a `bool`, as a number. +fn strict_integer_number(value: &Bound<'_, PyAny>) -> PyResult> { + if !value.is_instance_of::() || value.is_instance_of::() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +/// An `int`, including `bool`, which Python compares as one. +fn loose_integer(value: &Bound<'_, PyAny>) -> PyResult> { + if !value.is_instance_of::() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +fn number_value(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(Some(value.extract()?)); + } + Ok(None) +} + +/// `str(payload.get(key, default))`. +fn display_value(payload: &Bound<'_, PyAny>, key: &str, default: &str) -> PyResult { + let value: Bound<'_, PyAny> = payload.call_method1("get", (key, default))?; + Ok(value.str()?.to_string()) +} + +fn progress_payload(payload: &Bound<'_, PyAny>) -> PyResult { + let metadata: Bound<'_, PyAny> = payload_value(payload, "metadata")?; + let metadata_integer = |key: &str| -> PyResult> { + if metadata.downcast::().is_err() { + return Ok(None); + } + strict_integer(&payload_value(&metadata, key)?) + }; + Ok(ProgressPayload { + resource_name: text_value(&payload_value(payload, "resource_name")?)?, + resource_kind: text_value(&payload_value(payload, "resource_kind")?)?, + duration_ms: number_value(&payload_value(payload, "duration_ms")?)?, + attempt_number: loose_integer(&payload_value(payload, "attempt_number")?)?, + operation_name: text_value(&payload_value(payload, "operation_name")?)?, + metadata_attempt_number: metadata_integer("attempt_number")?, + metadata_item_count: metadata_integer("item_count")?, + metadata_unknown_count: metadata_integer("unknown_count")?, + error_type: text_value(&payload_value(payload, "error_type")?)?, + failed_attempt_number: strict_integer(&payload_value(payload, "failed_attempt_number")?)?, + next_attempt_number: strict_integer(&payload_value(payload, "next_attempt_number")?)?, + delay_ms: strict_integer_number(&payload_value(payload, "delay_ms")?)?, + query_id: text_value(&payload_value(payload, "query_id")?)?, + phase: text_value(&payload_value(payload, "phase")?)?, + intent: display_value(payload, "intent", DEFAULT_INTENT)?, + statement_kind: display_value(payload, "statement_kind", DEFAULT_STATEMENT_KIND)?, + }) +} + +fn optional_text(event: &Bound<'_, PyAny>, name: &str) -> PyResult> { + event.getattr(name)?.extract() +} + +/// The facts of one lifecycle event that progress projection reads. +fn progress_event(event: &Bound<'_, PyAny>) -> PyResult { + Ok(ProgressEvent { + event_id: event.getattr("event_id")?.extract()?, + event_type: event.getattr("event_type")?.extract()?, + resource_id: optional_text(event, "resource_id")?, + resource_attempt_id: optional_text(event, "resource_attempt_id")?, + operation_id: optional_text(event, "operation_id")?, + statement_id: optional_text(event, "statement_id")?, + payload: progress_payload(&event.getattr("payload")?)?, + }) +} + +/// The coordinator shared by every terminal writer in this process. +pub(crate) fn shared_coordinator(py: Python<'_>) -> PyResult> { + py.import(COORDINATOR_MODULE)? + .getattr(COORDINATOR_FUNCTION)? + .call0() +} + +fn persist(coordinator: &Bound<'_, PyAny>, stream: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + let keywords: Bound<'_, PyDict> = PyDict::new(coordinator.py()); + keywords.set_item("stream", stream)?; + keywords.set_item("text", text)?; + coordinator.call_method("write_persistent", (), Some(&keywords))?; + Ok(()) +} + +/// Write lasting text, clearing and then redrawing any live transient line. +pub(crate) fn write_persistent( + py: Python<'_>, + stream: &Bound<'_, PyAny>, + text: &str, +) -> PyResult<()> { + persist(&shared_coordinator(py)?, stream, text) +} + +pub(crate) fn terminal_stream(stream: &Bound<'_, PyAny>) -> PyResult { + if !stream.hasattr("isatty")? { + return Ok(false); + } + stream.call_method0("isatty")?.is_truthy() +} + +impl NativeProgressProjector { + fn locked(&self) -> PyResult> { + self.state + .lock() + .map_err(|_| PyRuntimeError::new_err("native progress projector lock is poisoned")) + } + + /// Run `update` and write its lines while holding the shared terminal line lock. + fn write_with( + &self, + py: Python<'_>, + update: impl FnOnce(ProgressProjection) -> (ProgressProjection, Vec, T), + ) -> PyResult { + let coordinator: &Bound<'_, PyAny> = self.coordinator.bind(py); + let lock: Bound<'_, PyAny> = coordinator.getattr("lock")?; + lock.call_method0("acquire")?; + let result: PyResult = (|| { + let (lines, value) = { + let mut state: MutexGuard<'_, ProgressProjection> = self.locked()?; + let (projection, lines, value) = update(std::mem::take(&mut *state)); + *state = projection; + (lines, value) + }; + for line in lines { + persist(coordinator, self.stream.bind(py), &format!("{line}\n"))?; + } + Ok(value) + })(); + lock.call_method0("release")?; + result + } +} + +/// The terminal a result callback names through its keyword arguments. +fn terminal_request( + py: Python<'_>, + function: &str, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, function, options, CLAIM_OPTIONS)?; + Ok(TerminalRequest { + resource_name: arguments.value("resource_name")?.extract()?, + resource_id: arguments.value("resource_id")?.extract()?, + resource_attempt_id: arguments.value("resource_attempt_id")?.extract()?, + attempt_number: arguments.value("attempt_number")?.extract()?, + }) +} + +#[pymethods] +impl NativeProgressProjector { + #[new] + #[pyo3(signature = (*, stream, use_color, debug = false))] + fn new( + py: Python<'_>, + stream: Bound<'_, PyAny>, + use_color: bool, + debug: bool, + ) -> PyResult { + let coordinator: Bound<'_, PyAny> = shared_coordinator(py)?; + Ok(Self { + state: Mutex::new(ProgressProjection::new(ProgressSettings { + use_color, + debug, + is_tty: terminal_stream(&stream)?, + })), + stream: stream.unbind(), + coordinator: coordinator.unbind(), + }) + } + + /// Install this projector for presentation adapters in the invocation context. + fn install<'py>(slf: &Bound<'py, Self>) -> PyResult> { + context_variable(slf.py())?.call_method1("set", (slf,)) + } + + /// Restore the previous invocation projector. + fn restore(&self, py: Python<'_>, token: &Bound<'_, PyAny>) -> PyResult<()> { + context_variable(py)?.call_method1("reset", (token,))?; + Ok(()) + } + + /// Add safe display order metadata without changing canonical facts. + #[pyo3(signature = (*, ordinals, total))] + fn configure_resources(&self, ordinals: &Bound<'_, PyAny>, total: i64) -> PyResult<()> { + let mut entries: Vec<(String, i64)> = Vec::new(); + for item in ordinals.call_method0("items")?.try_iter()? { + entries.push(item?.extract()?); + } + self.locked()?.configure_resources(entries, total); + Ok(()) + } + + /// Declare that an existing result callback will enrich this resource terminal. + #[pyo3(signature = (*, resource_name))] + fn expect_resource_enrichment(&self, resource_name: &str) -> PyResult<()> { + self.locked()?.expect_resource_enrichment(resource_name); + Ok(()) + } + + /// Return whether a visible canonical operation currently owns progress. + #[pyo3(name = "is_operation_active", signature = (*, operation_name))] + pub(crate) fn operation_active(&self, operation_name: &str) -> PyResult { + Ok(self.locked()?.is_operation_active(operation_name)) + } + + /// Consume one known lifecycle fact idempotently. + fn consume(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + let event: ProgressEvent = progress_event(event)?; + self.write_with(py, |projection| { + let (projection, lines) = consume_progress_event(projection, &event); + (projection, lines, ()) + }) + } + + /// Claim the targeted or latest terminal for one final resource result. + #[pyo3(signature = (**options))] + fn consume_resource_terminal( + &self, + py: Python<'_>, + options: Option>, + ) -> PyResult> { + let request: TerminalRequest = terminal_request(py, "consume_resource_terminal", options)?; + self.write_with(py, |projection| { + claim_progress_terminal(projection, &request) + }) + } + + /// Return a terminal duration without claiming its presentation. + #[pyo3(signature = (**options))] + fn resource_terminal_duration( + &self, + py: Python<'_>, + options: Option>, + ) -> PyResult> { + let request: TerminalRequest = terminal_request(py, "resource_terminal_duration", options)?; + Ok(progress_terminal_duration(&*self.locked()?, &request)) + } + + /// Render resource terminals no result callback claimed. + fn close(&self, py: Python<'_>) -> PyResult<()> { + self.write_with(py, |projection| { + let (projection, lines) = close_progress_projection(projection); + (projection, lines, ()) + }) + } +} + +fn context_variable(py: Python<'_>) -> PyResult> { + py.import(NATIVE_MODULE)?.getattr(CONTEXT_ATTRIBUTE) +} + +/// Return the projector installed for the current CLI invocation. +#[pyfunction] +fn current_native_progress_projector(py: Python<'_>) -> PyResult> { + context_variable(py)?.call_method0("get") +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("default", py.None())?; + module.add( + CONTEXT_ATTRIBUTE, + py.import("contextvars")? + .getattr("ContextVar")? + .call((CONTEXT_NAME,), Some(&keywords))?, + )?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(current_native_progress_projector, module)?) +} + +/// The projector installed for this invocation, when there is one. +pub(crate) fn current_projector(py: Python<'_>) -> PyResult>> { + let installed: Bound<'_, PyAny> = context_variable(py)?.call_method0("get")?; + if installed.is_none() { + return Ok(None); + } + Ok(Some(installed.extract()?)) +} + +/// Whether an installed projector shows a visible operation with this name. +pub(crate) fn projector_operation_active( + projector: &NativeProgressProjector, + operation_name: &str, +) -> PyResult { + projector.operation_active(operation_name) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs new file mode 100644 index 0000000000..e83139b0f1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs @@ -0,0 +1,212 @@ +//! Warehouse connection and planning phase progress reporters. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::progress::main::connection_phase_line::connection_phase_line; +use sqlbuild_core::progress::main::connection_start_message::connection_start_message; +use sqlbuild_core::progress::main::is_planning_completion_message::is_planning_completion_message; + +use crate::bindings::_helpers::cli::progress::{ + NativeProgressProjector, current_projector, projector_operation_active, write_persistent, +}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; + +const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; +const STATUS_REPORTER_CLASS: &str = "TransientStatusReporter"; +const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( + &["adapter_name", "stream"], + &[ + "blank_line_before_start", + "blank_line_after_complete", + "use_color", + ], +); +const PROJECT_COMPILE_OPERATION: &str = "project_compile"; + +/// Render simple connection start, success, and failure messages. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ConnectionProgressReporter { + adapter_name: String, + stream: Py, + blank_line_before_start: bool, + blank_line_after_complete: bool, + style: CliStyle, +} + +impl ConnectionProgressReporter { + fn write(&self, py: Python<'_>, message: &str) -> PyResult<()> { + write_persistent(py, self.stream.bind(py), &format!("{message}\n")) + } +} + +#[pymethods] +impl ConnectionProgressReporter { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "ConnectionProgressReporter", + options, + CONNECTION_OPTIONS, + )?; + Ok(Self { + adapter_name: arguments.value("adapter_name")?.extract()?, + stream: arguments.value("stream")?.unbind(), + blank_line_before_start: arguments.flag("blank_line_before_start")?, + blank_line_after_complete: arguments.flag("blank_line_after_complete")?, + style: CliStyle { + use_color: arguments.flag("use_color")?, + }, + }) + } + + fn on_connection_start(&self, py: Python<'_>, connection_count: i64) -> PyResult<()> { + if self.blank_line_before_start { + self.write(py, "")?; + } + let message: String = connection_start_message(&self.adapter_name, connection_count); + self.write(py, &self.style.muted(&message)) + } + + #[pyo3(signature = (*, connection_count, elapsed_seconds))] + fn on_connection_complete( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + let _ = connection_count; + let line: String = + connection_phase_line(self.style, true, (&self.adapter_name, elapsed_seconds)); + self.write(py, &line)?; + if self.blank_line_after_complete { + self.write(py, "")?; + } + Ok(()) + } + + #[pyo3(signature = (*, connection_count, elapsed_seconds))] + fn on_connection_error( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + let _ = connection_count; + let line: String = + connection_phase_line(self.style, false, (&self.adapter_name, elapsed_seconds)); + self.write(py, &line) + } +} + +/// Render simple planning phase progress messages. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct PlanningProgressReporter { + status: Py, + active: AtomicBool, + projector: Option>, +} + +impl PlanningProgressReporter { + fn call_status(&self, py: Python<'_>, method: &str, message: &str) -> PyResult<()> { + self.status.bind(py).call_method1(method, (message,))?; + Ok(()) + } + + fn call_status_keyword(&self, py: Python<'_>, method: &str, message: &str) -> PyResult<()> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("message", message)?; + self.status + .bind(py) + .call_method(method, (), Some(&keywords))?; + Ok(()) + } +} + +#[pymethods] +impl PlanningProgressReporter { + #[new] + #[pyo3(signature = (*, stream, use_color = false))] + fn new(py: Python<'_>, stream: &Bound<'_, PyAny>, use_color: bool) -> PyResult { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("stream", stream)?; + keywords.set_item("use_color", use_color)?; + let status: Bound<'_, PyAny> = py + .import(STATUS_REPORTER_MODULE)? + .getattr(STATUS_REPORTER_CLASS)? + .call((), Some(&keywords))?; + Ok(Self { + status: status.unbind(), + active: AtomicBool::new(false), + projector: current_projector(py)?, + }) + } + + fn on_progress(&self, py: Python<'_>, message: &str) -> PyResult<()> { + if let Some(projector) = &self.projector + && projector_operation_active(projector.get(), PROJECT_COMPILE_OPERATION)? + { + return Ok(()); + } + if is_planning_completion_message(message) { + return self.complete(py, message); + } + if !self.active.load(Ordering::SeqCst) { + return self.start(py, message); + } + self.update(py, message) + } + + fn start(&self, py: Python<'_>, message: &str) -> PyResult<()> { + self.call_status(py, "start", message)?; + self.active.store(true, Ordering::SeqCst); + Ok(()) + } + + fn update(&self, py: Python<'_>, message: &str) -> PyResult<()> { + if !self.active.load(Ordering::SeqCst) { + return self.start(py, message); + } + self.call_status(py, "update", message) + } + + fn complete(&self, py: Python<'_>, message: &str) -> PyResult<()> { + self.call_status_keyword(py, "complete", message)?; + self.active.store(false, Ordering::SeqCst); + Ok(()) + } + + /// Complete with a pre-styled message written verbatim. + fn complete_styled(&self, py: Python<'_>, message: &str) -> PyResult<()> { + self.call_status_keyword(py, "complete_styled", message)?; + self.active.store(false, Ordering::SeqCst); + Ok(()) + } + + /// Close active progress and render a non-success terminal message. + fn error(&self, py: Python<'_>, message: &str) -> PyResult<()> { + self.call_status(py, "error", message)?; + self.active.store(false, Ordering::SeqCst); + Ok(()) + } + + #[pyo3(signature = (*, blank_line_after = false))] + fn finish(&self, py: Python<'_>, blank_line_after: bool) -> PyResult<()> { + self.status.bind(py).call_method0("close")?; + self.active.store(false, Ordering::SeqCst); + if blank_line_after { + self.status.bind(py).call_method0("write_blank_line")?; + } + Ok(()) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs new file mode 100644 index 0000000000..f5e80e4a5f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs @@ -0,0 +1,170 @@ +//! Execution headers, SQL expectation rows and verbose inspection lines of CLI progress. + +use std::sync::{Mutex, MutexGuard}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::progress::main::execution_header_text::execution_header_text; +use sqlbuild_core::progress::main::format_execution_header::format_execution_header as format_header; +use sqlbuild_core::progress::main::format_expectation_detail::format_expectation_detail as format_detail; +use sqlbuild_core::progress::main::format_expectation_name::format_expectation_name as format_name; +use sqlbuild_core::progress::main::format_inspection_record::format_inspection_record; +use sqlbuild_core::progress::models::{ExpectationStep, InspectionRecord}; +use sqlbuild_core::progress::types::DifferenceSample; + +use crate::bindings::_helpers::cli::progress::write_persistent; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; + +const HEADER_OPTIONS: (&[&str], &[&str]) = ( + &["stream", "command", "target", "concurrency", "use_color"], + &[], +); + +/// Format command and execution context for a progress header. +#[pyfunction] +#[pyo3(signature = (*, command, target, concurrency))] +fn format_execution_header(command: &str, target: Option<&str>, concurrency: i64) -> String { + format_header(command, target, concurrency) +} + +/// Write an execution header for command progress output. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_execution_header(py: Python<'_>, options: Option>) -> PyResult<()> { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "write_execution_header", options, HEADER_OPTIONS)?; + let command: String = arguments.value("command")?.extract()?; + let target: Option = arguments.value("target")?.extract()?; + let concurrency: i64 = arguments.value("concurrency")?.extract()?; + let style: CliStyle = CliStyle { + use_color: arguments.value("use_color")?.extract()?, + }; + let text: String = execution_header_text(style, (&command, target.as_deref(), concurrency)); + let stream: Bound<'_, PyAny> = arguments.value("stream")?; + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + Ok(()) +} + +/// Format a comparison result name as a user-facing expectation name. +#[pyfunction] +fn format_expectation_name(model_name: &str) -> String { + format_name(model_name) +} + +fn difference_samples(samples: &Bound<'_, PyAny>) -> PyResult> { + let mut rows: Vec = Vec::new(); + for sample in samples.try_iter()? { + rows.push(sample?.getattr("values")?.extract()?); + } + Ok(rows) +} + +fn expectation_step(step: &Bound<'_, PyAny>) -> PyResult { + let mut column_differences: Vec<(String, String, String)> = Vec::new(); + for difference in step.getattr("column_differences")?.try_iter()? { + let difference: Bound<'_, PyAny> = difference?; + column_differences.push(( + difference.getattr("name")?.str()?.to_string(), + difference.getattr("actual")?.str()?.to_string(), + difference.getattr("expected")?.str()?.to_string(), + )); + } + Ok(ExpectationStep { + model_name: step.getattr("model_name")?.extract()?, + outcome: step.getattr("outcome")?.str()?.to_string(), + actual_row_count: step.getattr("actual_row_count")?.extract()?, + expected_row_count: step.getattr("expected_row_count")?.extract()?, + mismatched_row_count: step.getattr("mismatched_row_count")?.extract()?, + unexpected_row_count: step.getattr("unexpected_row_count")?.extract()?, + missing_row_count: step.getattr("missing_row_count")?.extract()?, + unexpected_samples: difference_samples(&step.getattr("unexpected_samples")?)?, + missing_samples: difference_samples(&step.getattr("missing_samples")?)?, + column_differences, + }) +} + +/// Format failing row count detail for an expectation row. +#[pyfunction] +fn format_expectation_detail(step_result: &Bound<'_, PyAny>) -> PyResult { + Ok(format_detail(&expectation_step(step_result)?)) +} + +fn inspection_record(record: &Bound<'_, PyAny>) -> PyResult { + Ok(InspectionRecord { + sql: record.getattr("sql")?.extract()?, + elapsed_seconds: record.getattr("elapsed_seconds")?.extract()?, + row_count: record.getattr("row_count")?.extract()?, + error: record.getattr("error")?.extract()?, + }) +} + +/// Write each inspection read as one truncated stderr line and keep a running total. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct VerboseInspectionWriter { + totals: Mutex<(usize, f64)>, +} + +impl VerboseInspectionWriter { + fn locked(&self) -> PyResult> { + self.totals + .lock() + .map_err(|_| PyRuntimeError::new_err("inspection writer lock is poisoned")) + } +} + +fn write_stderr_line(py: Python<'_>, text: &str) -> PyResult<()> { + let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; + write_persistent(py, &stderr, &format!("{text}\n")) +} + +#[pymethods] +impl VerboseInspectionWriter { + #[new] + fn new() -> Self { + Self { + totals: Mutex::new((0, 0.0)), + } + } + + /// Write one inspection read; safe to call from inspection worker threads. + fn write(&self, py: Python<'_>, record: &Bound<'_, PyAny>) -> PyResult<()> { + let record: InspectionRecord = inspection_record(record)?; + { + let mut totals: MutexGuard<'_, (usize, f64)> = self.locked()?; + totals.0 += 1; + totals.1 += record.elapsed_seconds; + } + write_stderr_line(py, &format_inspection_record(&record)) + } + + /// Write the read count and cumulative query time when anything was read. + fn write_total(&self, py: Python<'_>) -> PyResult<()> { + let (count, seconds) = *self.locked()?; + if count == 0 { + return Ok(()); + } + write_stderr_line( + py, + &format!("Inspection queries: {count} ({seconds:.2}s cumulative query time)"), + ) + } + + /// Render one inspection read as a single truncated diagnostic line. + #[staticmethod] + #[pyo3(signature = (*, record))] + fn format_record(record: &Bound<'_, PyAny>) -> PyResult { + Ok(format_inspection_record(&inspection_record(record)?)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(format_execution_header, module)?)?; + module.add_function(wrap_pyfunction!(write_execution_header, module)?)?; + module.add_function(wrap_pyfunction!(format_expectation_name, module)?)?; + module.add_function(wrap_pyfunction!(format_expectation_detail, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 6868fea589..5c21cfe7a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -2,10 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cli::{argv, entry}; +use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { argv::register(module)?; entry::register(module)?; + progress::register(module)?; + progress_reporters::register(module)?; + progress_text::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs new file mode 100644 index 0000000000..7ee3a0cc7d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -0,0 +1,336 @@ +//! Connection config resolution for CLI commands. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::command_connection::constants::{ + CONNECTION_DATABASE_KEY, CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, + DBT_PROFILE_ROUTING_KEYS, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, PROJECT_CONFIG_FILENAME, +}; +use sqlbuild_config::command_connection::main::missing_connection_error::missing_connection_error; +use sqlbuild_config::command_connection::main::snowflake_key_warning::snowflake_key_warning; + +use crate::bindings::_helpers::command_runtime::runtime_errors::raised; +use crate::bindings::_helpers::command_runtime::warehouses::{WarehouseInputs, apply_warehouse}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const DBT_PROFILE_MODULE: &str = "sqlbuild.integrations.dbt.main.profile.profile_connection"; +const DBT_ERRORS_MODULE: &str = "sqlbuild.integrations.dbt.exceptions"; +const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( + &["raw_config", "project_dir", "adapter_name"], + &["discovered_inputs", "cli_vars", "selected_target"], +); +const PROJECT_CONNECTION_OPTIONS: (&[&str], &[&str]) = ( + &["discovered_inputs", "project_dir"], + &["selected_target", "cli_vars", "require_connection"], +); +const TARGET_CONNECTION_OPTIONS: (&[&str], &[&str]) = ( + &["discovered_inputs", "project_dir", "target_name"], + &["cli_vars"], +); +const DBT_PROFILE_REQUIRES_PROJECT: &str = + "connection.source = 'dbt_profile' requires project configuration"; + +fn path<'py>(py: Python<'py>, value: &Bound<'py, PyAny>) -> PyResult> { + imported(py, "pathlib", "Path")?.call1((value,)) +} + +fn warn(py: Python<'_>, message: &str) -> PyResult<()> { + py.import("sys")? + .getattr("stderr")? + .call_method1("write", (format!("{message}\n"),))?; + Ok(()) +} + +fn dbt_profile_error(py: Python<'_>) -> PyErr { + match imported(py, DBT_ERRORS_MODULE, "DbtProfileError") + .and_then(|class| class.call1((DBT_PROFILE_REQUIRES_PROJECT,))) + { + Ok(instance) => PyErr::from_value(instance), + Err(error) => error, + } +} + +/// The config with a referenced dbt profile resolved and user overrides kept on top. +fn with_dbt_profile<'py>( + raw_config: &Bound<'py, PyDict>, + config: Bound<'py, PyDict>, + (project_dir, adapter_name): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (discovered_inputs, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = raw_config.py(); + let profile: Bound<'py, PyAny> = call_with( + &imported(py, DBT_PROFILE_MODULE, "resolve_raw_dbt_profile_connection")?, + &[ + ("raw_config", config.clone().into_any()), + ("project_dir", project_dir.clone()), + ("adapter_name", adapter_name.clone()), + ( + "project_config", + discovered_inputs.getattr("project_config")?, + ), + ("cli_vars", cli_vars.clone()), + ], + )?; + if profile.is_none() { + return Ok(config); + } + for warning in profile.getattr("warnings")?.try_iter()? { + warn(py, &format!("Warning: {}", warning?.str()?))?; + } + let merged: Bound<'py, PyDict> = PyDict::new(py); + merged.update(&profile.getattr("connection")?.cast_into()?)?; + for (key, value) in raw_config.iter() { + let routing: bool = key + .extract::() + .is_ok_and(|key| DBT_PROFILE_ROUTING_KEYS.contains(&key.as_str())); + if !routing { + merged.set_item(key, value)?; + } + } + Ok(merged) +} + +/// Make a relative DuckDB database path relative to the project directory. +fn anchor_duckdb_database( + config: &Bound<'_, PyDict>, + (project_dir, adapter_name): (&Bound<'_, PyAny>, &str), +) -> PyResult<()> { + let py: Python<'_> = config.py(); + if adapter_name != DUCKDB_ADAPTER { + return Ok(()); + } + let Some(database) = config.get_item(CONNECTION_DATABASE_KEY)? else { + return Ok(()); + }; + if !database.is_instance_of::() { + return Ok(()); + } + if path(py, &database)? + .call_method0("is_absolute")? + .is_truthy()? + || database.extract::()? == DUCKDB_MEMORY_DATABASE + { + return Ok(()); + } + let anchored: Bound<'_, PyAny> = path(py, project_dir)?.div(&database)?.str()?.into_any(); + config.set_item(CONNECTION_DATABASE_KEY, anchored) +} + +/// Resolve dbt profiles, relative DuckDB paths, and the command group's warehouse. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_connection_config<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "resolve_connection_config", options, CONNECTION_OPTIONS)?; + connection_config( + &arguments.value("raw_config")?.cast_into()?, + ( + &arguments.value("project_dir")?, + &arguments.value("adapter_name")?, + ), + ( + &arguments.value("discovered_inputs")?, + &arguments.value("cli_vars")?, + ), + &arguments.value("selected_target")?, + ) +} + +fn connection_config<'py>( + raw_config: &Bound<'py, PyDict>, + (project_dir, adapter_name): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (discovered_inputs, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + selected_target: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = raw_config.py(); + let mut config: Bound<'py, PyDict> = raw_config.copy()?; + if !discovered_inputs.is_none() { + config = with_dbt_profile( + raw_config, + config, + (project_dir, adapter_name), + (discovered_inputs, cli_vars), + )?; + } else if config + .get_item(CONNECTION_SOURCE_KEY)? + .map(|source| source.eq(DBT_PROFILE_CONNECTION_SOURCE)) + .transpose()? + .unwrap_or(false) + { + return Err(dbt_profile_error(py)); + } + let adapter: String = adapter_name.str()?.extract()?; + let mut keys: Vec = Vec::new(); + for key in config.keys() { + if key.is_instance_of::() { + keys.push(key.extract()?); + } + } + if let Some(warning) = snowflake_key_warning(&adapter, &keys) { + warn(py, &warning)?; + } + anchor_duckdb_database(&config, (project_dir, &adapter))?; + apply_warehouse( + config.as_any(), + (adapter_name, project_dir), + (!discovered_inputs.is_none()).then_some(WarehouseInputs { + discovered_inputs, + selected_target, + }), + ) +} + +/// Reject warehouse commands whose connection would come from no configuration at all. +fn require_configured_connection( + discovered_inputs: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, + selected_target: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = project_dir.py(); + let project_config: Bound<'_, PyAny> = discovered_inputs.getattr("project_config")?; + let local_config: Bound<'_, PyAny> = discovered_inputs.getattr("local_config")?; + if project_config.getattr("connection")?.is_truthy()? + || local_config.getattr("connection")?.is_truthy()? + { + return Ok(()); + } + let target_name: Bound<'_, PyAny> = call_with( + &imported(py, TARGET_NAME_MODULE, "resolve_target_name")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("selected_target", selected_target.clone()), + ], + )?; + let project_file: String = path(py, project_dir)? + .div(PROJECT_CONFIG_FILENAME)? + .str()? + .extract()?; + if target_name.is_none() { + return Err(raised(py, missing_connection_error(&project_file, None))); + } + let target_config: Bound<'_, PyAny> = call_with( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ("project_config", project_config), + ("local_config", local_config), + ("target_name", target_name.clone()), + ], + )?; + if !target_config.getattr("connection_name")?.is_none() + || target_config.getattr("connection")?.is_truthy()? + { + return Ok(()); + } + let target_name: String = target_name.str()?.extract()?; + Err(raised( + py, + missing_connection_error(&project_file, Some(&target_name)), + )) +} + +pub(crate) fn project_connection_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, + (project_dir, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (cli_vars, require_connection): (&Bound<'py, PyAny>, bool), +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + if require_connection { + require_configured_connection(discovered_inputs, project_dir, selected_target)?; + } + let raw_config: Bound<'py, PyDict> = call_with( + &imported( + py, + EFFECTIVE_CONFIG_MODULE, + "build_effective_connection_config", + )?, + &[ + ("discovered_inputs", discovered_inputs.clone()), + ("selected_target", selected_target.clone()), + ("cli_vars", cli_vars.clone()), + ], + )? + .cast_into()?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ( + "project_config", + discovered_inputs.getattr("project_config")?, + ), + ("local_config", discovered_inputs.getattr("local_config")?), + ], + )?; + connection_config( + &raw_config, + (project_dir, &adapter_name), + (discovered_inputs, cli_vars), + selected_target, + ) +} + +/// Resolve the project connection; offline modes pass `require_connection=False`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_project_connection_config<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_project_connection_config", + options, + PROJECT_CONNECTION_OPTIONS, + )?; + let require_connection: bool = + !arguments.passed("require_connection")? || arguments.flag("require_connection")?; + project_connection_config( + &arguments.value("discovered_inputs")?, + ( + &arguments.value("project_dir")?, + &arguments.value("selected_target")?, + ), + (&arguments.value("cli_vars")?, require_connection), + ) +} + +/// Resolve the effective connection config for one named target. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_target_connection_config<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_target_connection_config", + options, + TARGET_CONNECTION_OPTIONS, + )?; + project_connection_config( + &arguments.value("discovered_inputs")?, + ( + &arguments.value("project_dir")?, + &arguments.value("target_name")?, + ), + (&arguments.value("cli_vars")?, true), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_connection_config, module)?)?; + module.add_function(wrap_pyfunction!(resolve_project_connection_config, module)?)?; + module.add_function(wrap_pyfunction!(resolve_target_connection_config, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs new file mode 100644 index 0000000000..df3675c30b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod connection; +pub(crate) mod preview_target; +pub(crate) mod registration; +pub(crate) mod runtime_errors; +pub(crate) mod warehouses; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs new file mode 100644 index 0000000000..0bfa4096cf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs @@ -0,0 +1,38 @@ +//! Shared validation for inspection-only `--as` target previews. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::command_connection::main::check_preview_target::check_preview_target; + +use crate::bindings::_helpers::command_runtime::runtime_errors::raised; + +fn target_names(config: &Bound<'_, PyAny>) -> PyResult> { + config + .getattr("targets")? + .try_iter()? + .map(|name| name?.extract()) + .collect() +} + +/// Fail clearly when `--as` names a target that is not configured. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, as_target, command_name))] +fn validate_preview_target( + py: Python<'_>, + discovered_inputs: Bound<'_, PyAny>, + as_target: Option, + command_name: String, +) -> Result<(), PyErr> { + let Some(as_target) = as_target else { + return Ok(()); + }; + let mut configured: Vec = target_names(&discovered_inputs.getattr("project_config")?)?; + configured.extend(target_names(&discovered_inputs.getattr("local_config")?)?); + check_preview_target(&as_target, &command_name, &configured).map_err(|error| raised(py, error)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(validate_preview_target, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs new file mode 100644 index 0000000000..9aa9b25ee4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs @@ -0,0 +1,11 @@ +//! Register command connection and warehouse resolution. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::command_runtime::{connection, preview_target, warehouses}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + warehouses::register(module)?; + connection::register(module)?; + preview_target::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs new file mode 100644 index 0000000000..a20f23f817 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs @@ -0,0 +1,55 @@ +//! The Python exceptions command connection failures are raised as. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_config::command_connection::errors::CommandConnectionError; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const CLI_ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; + +/// The `CliUserError` or `ProjectConfigError` for `error`. +pub(crate) fn raised(py: Python<'_>, error: CommandConnectionError) -> PyErr { + let created = || -> PyResult { + let instance: Bound<'_, PyAny> = match error { + CommandConnectionError::User { + code, + message, + help, + } => { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", code)?; + keywords.set_item("help", help)?; + imported(py, CLI_ERRORS_MODULE, "CliUserError")? + .call((message,), Some(&keywords))? + } + CommandConnectionError::ProjectConfig(message) => { + imported(py, DISCOVERY_ERRORS_MODULE, "ProjectConfigError")?.call1((message,))? + } + }; + Ok(PyErr::from_value(instance)) + }; + created().unwrap_or_else(|failure| failure) +} + +/// Whether `error` is a `CompileInputError` or a `CliUserError`, the failures resolution reports. +pub(crate) fn expected_resolution_error(py: Python<'_>, error: &PyErr) -> PyResult { + Ok( + error.is_instance(py, &imported(py, CLI_ERRORS_MODULE, "CliUserError")?) + || error.is_instance( + py, + &imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?, + ), + ) +} + +/// The message a resolution error shows: `CliUserError.message`, else `str(error)`. +pub(crate) fn resolution_message(py: Python<'_>, error: &PyErr) -> PyResult { + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + if error.is_instance(py, &imported(py, CLI_ERRORS_MODULE, "CliUserError")?) { + return value.getattr("message")?.str()?.extract(); + } + value.str()?.extract() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs new file mode 100644 index 0000000000..fe3b51108a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -0,0 +1,487 @@ +//! Command-group warehouse selection for CLI warehouse connections. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::command_connection::constants::{ + CLI_WAREHOUSE_SOURCE, CONNECTION_DEFAULT_WAREHOUSE_SOURCE, CONNECTION_WAREHOUSE_KEY, + CONNECTION_WAREHOUSE_SOURCE, +}; +use sqlbuild_config::command_connection::main::authored_warehouse::authored_warehouse; +use sqlbuild_config::command_connection::main::check_cli_warehouse::check_cli_warehouse; +use sqlbuild_config::command_connection::main::check_expanded_warehouse::check_expanded_warehouse; +use sqlbuild_config::command_connection::main::unsupported_cli_warehouse::unsupported_cli_warehouse; +use sqlbuild_config::command_connection::models::AuthoredWarehouse; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::bindings::_helpers::command_runtime::runtime_errors::{ + expected_resolution_error, raised, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; +const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main.session_warehouse_support"; +const DISCOVER_CONFIGURATION_MODULE: &str = + "sqlbuild.compiler.discovery.main.discover_configuration"; +const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; +const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild.compiler.compile.main.expand_template_data"; +const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( + &[ + "discovered_inputs", + "selected_target", + "group", + "cli_warehouse", + "connection_config", + ], + &["cli_vars"], +); +const APPLY_OPTIONS: (&[&str], &[&str]) = ( + &[ + "config", + "adapter_name", + "project_dir", + "discovered_inputs", + "selected_target", + ], + &[], +); + +/// The active command's warehouse group, `--warehouse` value and CLI vars. +pub(crate) struct CommandScope<'py> { + pub(crate) group: Bound<'py, PyAny>, + pub(crate) cli_warehouse: Option, + pub(crate) cli_vars: Bound<'py, PyAny>, +} + +/// A target's authored group warehouse with the `WarehouseGroup` it was found for. +pub(crate) struct FoundWarehouse<'py> { + pub(crate) authored: AuthoredWarehouse, + pub(crate) group: Bound<'py, PyAny>, +} + +/// The project inputs and target a warehouse is resolved against. +#[derive(Clone, Copy)] +pub(crate) struct WarehouseInputs<'a, 'py> { + pub(crate) discovered_inputs: &'a Bound<'py, PyAny>, + pub(crate) selected_target: &'a Bound<'py, PyAny>, +} + +fn none(py: Python<'_>) -> Bound<'_, PyAny> { + py.None().into_bound(py) +} + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +pub(crate) fn command_scope(py: Python<'_>) -> PyResult> { + let scope: Bound<'_, PyAny> = + imported(py, SCOPE_MODULE, "current_command_warehouse_scope")?.call0()?; + Ok(CommandScope { + group: scope.getattr("group")?, + cli_warehouse: optional_text(&scope.getattr("cli_warehouse")?)?, + cli_vars: scope.getattr("cli_vars")?, + }) +} + +/// `ResolvedWarehouse(warehouse=..., source=...)`. +fn resolved_warehouse<'py>( + py: Python<'py>, + warehouse: Option<&str>, + source: &str, +) -> PyResult> { + call_with( + &imported(py, ENTRY_MODELS, "ResolvedWarehouse")?, + &[ + ( + "warehouse", + match warehouse { + Some(warehouse) => PyString::new(py, warehouse).into_any(), + None => none(py), + }, + ), + ("source", PyString::new(py, source).into_any()), + ], + ) +} + +fn group_value(target: &Bound<'_, PyAny>, group: &Bound<'_, PyAny>) -> PyResult> { + if target.is_none() { + return Ok(None); + } + optional_text( + &target + .getattr("warehouses")? + .call_method1("for_group", (group,))?, + ) +} + +/// The active target's authored, unexpanded warehouse for one command group. +pub(crate) fn find_authored<'py>( + inputs: WarehouseInputs<'_, 'py>, + group: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = group.py(); + let project_config: Bound<'py, PyAny> = inputs.discovered_inputs.getattr("project_config")?; + let local_config: Bound<'py, PyAny> = inputs.discovered_inputs.getattr("local_config")?; + let target_name: Bound<'py, PyAny> = call_with( + &imported(py, TARGET_NAME_MODULE, "resolve_target_name")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("selected_target", inputs.selected_target.clone()), + ], + )?; + if target_name.is_none() { + return Ok(None); + } + let local_target: Bound<'py, PyAny> = local_config + .getattr("targets")? + .call_method1("get", (&target_name,))?; + let project_target: Bound<'py, PyAny> = project_config + .getattr("targets")? + .call_method1("get", (&target_name,))?; + let target_name: String = target_name.extract()?; + let group_name: String = group.getattr("value")?.extract()?; + Ok(authored_warehouse( + (&target_name, &group_name), + group_value(&local_target, group)?, + group_value(&project_target, group)?, + ) + .map(|authored| FoundWarehouse { + authored, + group: group.clone(), + })) +} + +/// The group warehouse with `${...}` templates expanded, checked as an identifier. +fn expand_authored( + found: &FoundWarehouse<'_>, + inputs: WarehouseInputs<'_, '_>, + cli_vars: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = found.group.py(); + let runtime: Bound<'_, PyAny> = call_with( + &imported( + py, + EFFECTIVE_RUNTIME_MODULE, + "build_effective_runtime_config", + )?, + &[ + ("discovered_inputs", inputs.discovered_inputs.clone()), + ("selected_target", inputs.selected_target.clone()), + ("cli_vars", cli_vars.clone()), + ], + )?; + let expanded: Bound<'_, PyAny> = call_with( + &imported(py, EXPAND_TEMPLATE_MODULE, "expand_template_data")?, + &[ + ( + "value", + PyString::new(py, &found.authored.warehouse).into_any(), + ), + ("variables", runtime.get_item(1)?), + ("context_values", PyDict::new(py).into_any()), + ( + "context_label", + PyString::new(py, &found.authored.context_label()).into_any(), + ), + ( + "allow_context", + PyBool::new(py, false).to_owned().into_any(), + ), + ( + "preserve_context_tokens", + PyBool::new(py, false).to_owned().into_any(), + ), + ( + "preserve_unknown_context", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?; + let warehouse: String = expanded.str()?.extract()?; + check_expanded_warehouse(&found.authored, &warehouse).map_err(|error| raised(py, error))?; + Ok(warehouse) +} + +fn connection_warehouse(connection_config: &Bound<'_, PyAny>) -> PyResult> { + let value: Bound<'_, PyAny> = + connection_config.call_method1("get", (CONNECTION_WAREHOUSE_KEY,))?; + if !value.is_instance_of::() { + return Ok(None); + } + let text: String = value.extract()?; + Ok((!python_strip(&text).is_empty()).then_some(text)) +} + +fn session_warehouse_support( + py: Python<'_>, + adapter_name: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + call_with( + &imported( + py, + SESSION_WAREHOUSE_MODULE, + "adapter_supports_session_warehouse", + )?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", project_dir.clone()), + ], + )? + .is_truthy() +} + +fn checked_cli_warehouse(py: Python<'_>, cli_warehouse: &str) -> PyResult<()> { + check_cli_warehouse(cli_warehouse).map_err(|error| raised(py, error)) +} + +fn unsupported(py: Python<'_>, cli_warehouse: &str, adapter_name: &Bound<'_, PyAny>) -> PyErr { + match adapter_name.str().and_then(|name| name.extract::()) { + Ok(name) => raised(py, unsupported_cli_warehouse(cli_warehouse, &name)), + Err(error) => error, + } +} + +/// One group's warehouse and its source: CLI, then target group, then connection, then default. +pub(crate) fn resolve_warehouse( + inputs: Option>, + (group, cli_warehouse): (&Bound<'_, PyAny>, Option<&str>), + (connection_config, cli_vars): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<(Option, String)> { + let py: Python<'_> = group.py(); + if let Some(cli_warehouse) = cli_warehouse { + checked_cli_warehouse(py, cli_warehouse)?; + return Ok(( + Some(cli_warehouse.to_owned()), + CLI_WAREHOUSE_SOURCE.to_owned(), + )); + } + if let Some(inputs) = inputs + && !group.is_none() + && let Some(found) = find_authored(inputs, group)? + { + let warehouse: String = expand_authored(&found, inputs, cli_vars)?; + return Ok((Some(warehouse), found.authored.source())); + } + Ok(match connection_warehouse(connection_config)? { + Some(warehouse) => (Some(warehouse), CONNECTION_WAREHOUSE_SOURCE.to_owned()), + None => (None, CONNECTION_DEFAULT_WAREHOUSE_SOURCE.to_owned()), + }) +} + +fn optional_inputs<'a, 'py>( + discovered_inputs: &'a Bound<'py, PyAny>, + selected_target: &'a Bound<'py, PyAny>, +) -> Option> { + (!discovered_inputs.is_none()).then_some(WarehouseInputs { + discovered_inputs, + selected_target, + }) +} + +/// Reject an empty, malformed, or unsupported `--warehouse` before the command runs. +#[pyfunction] +#[pyo3(signature = (*, cli_warehouse, project_dir))] +fn validate_cli_warehouse( + py: Python<'_>, + cli_warehouse: Option, + project_dir: Bound<'_, PyAny>, +) -> Result<(), PyErr> { + let Some(cli_warehouse) = cli_warehouse else { + return Ok(()); + }; + checked_cli_warehouse(py, &cli_warehouse)?; + let discovered: Bound<'_, PyAny> = call_with( + &imported( + py, + DISCOVER_CONFIGURATION_MODULE, + "discover_project_configuration", + )?, + &[("project_dir", project_dir.clone())], + )?; + let adapter_name: Bound<'_, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + if !session_warehouse_support(py, &adapter_name, &project_dir)? { + return Err(unsupported(py, &cli_warehouse, &adapter_name)); + } + Ok(()) +} + +/// Find the active target's authored, unexpanded warehouse for one command group. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target, group))] +fn find_target_group_warehouse<'py>( + py: Python<'py>, + discovered_inputs: Bound<'py, PyAny>, + selected_target: Bound<'py, PyAny>, + group: Bound<'py, PyAny>, +) -> PyResult> { + let inputs: WarehouseInputs<'_, 'py> = WarehouseInputs { + discovered_inputs: &discovered_inputs, + selected_target: &selected_target, + }; + let Some(found) = find_authored(inputs, &group)? else { + return Ok(none(py)); + }; + call_with( + &imported(py, ENTRY_MODELS, "AuthoredTargetWarehouse")?, + &[ + ( + "warehouse", + PyString::new(py, &found.authored.warehouse).into_any(), + ), + ( + "target_name", + PyString::new(py, &found.authored.target_name).into_any(), + ), + ("group", found.group), + ( + "file_name", + PyString::new(py, &found.authored.file_name).into_any(), + ), + ], + ) +} + +/// Resolve one group's warehouse: CLI, then target group, then connection, then default. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_command_warehouse<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "resolve_command_warehouse", options, RESOLVE_OPTIONS)?; + let discovered_inputs: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; + let selected_target: Bound<'py, PyAny> = arguments.value("selected_target")?; + let cli_warehouse: Option = optional_text(&arguments.value("cli_warehouse")?)?; + let (warehouse, source) = resolve_warehouse( + optional_inputs(&discovered_inputs, &selected_target), + (&arguments.value("group")?, cli_warehouse.as_deref()), + ( + &arguments.value("connection_config")?, + &arguments.value("cli_vars")?, + ), + )?; + resolved_warehouse(py, warehouse.as_deref(), &source) +} + +/// The connection config with the active command group's warehouse applied. +pub(crate) fn apply_warehouse<'py>( + config: &Bound<'py, PyAny>, + (adapter_name, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + inputs: Option>, +) -> PyResult> { + let py: Python<'py> = config.py(); + let scope: CommandScope<'py> = command_scope(py)?; + let authored: bool = match inputs { + Some(inputs) if !scope.group.is_none() => find_authored(inputs, &scope.group)?.is_some(), + _ => false, + }; + if scope.cli_warehouse.is_none() && !authored { + return Ok(config.clone()); + } + if !session_warehouse_support(py, adapter_name, project_dir)? { + return match &scope.cli_warehouse { + Some(cli_warehouse) => Err(unsupported(py, cli_warehouse, adapter_name)), + None => Ok(config.clone()), + }; + } + let (warehouse, source) = resolve_warehouse( + inputs, + (&scope.group, scope.cli_warehouse.as_deref()), + (config, &scope.cli_vars), + )?; + if source == CONNECTION_WAREHOUSE_SOURCE || source == CONNECTION_DEFAULT_WAREHOUSE_SOURCE { + return Ok(config.clone()); + } + let applied: Bound<'py, PyDict> = PyDict::new(py); + applied.update(&config.cast::()?.clone())?; + applied.set_item(CONNECTION_WAREHOUSE_KEY, warehouse)?; + Ok(applied.into_any()) +} + +/// Return connection config whose warehouse follows the active command's group. +#[pyfunction] +#[pyo3(signature = (**options))] +fn apply_command_warehouse<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "apply_command_warehouse", options, APPLY_OPTIONS)?; + let discovered_inputs: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; + let selected_target: Bound<'py, PyAny> = arguments.value("selected_target")?; + apply_warehouse( + &arguments.value("config")?, + ( + &arguments.value("adapter_name")?, + &arguments.value("project_dir")?, + ), + optional_inputs(&discovered_inputs, &selected_target), + ) +} + +/// Describe the warehouse an already resolved connection uses and where it came from. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target, connection_config))] +fn describe_connection_warehouse<'py>( + py: Python<'py>, + discovered_inputs: Bound<'py, PyAny>, + selected_target: Bound<'py, PyAny>, + connection_config: Bound<'py, PyAny>, +) -> PyResult> { + let scope: CommandScope<'py> = command_scope(py)?; + let Some(warehouse) = connection_warehouse(&connection_config)? else { + return resolved_warehouse(py, None, CONNECTION_DEFAULT_WAREHOUSE_SOURCE); + }; + if scope.cli_warehouse.as_deref() == Some(warehouse.as_str()) { + return resolved_warehouse(py, Some(&warehouse), CLI_WAREHOUSE_SOURCE); + } + let inputs: WarehouseInputs<'_, 'py> = WarehouseInputs { + discovered_inputs: &discovered_inputs, + selected_target: &selected_target, + }; + let found: Option> = if scope.group.is_none() { + None + } else { + find_authored(inputs, &scope.group)? + }; + if let Some(found) = found { + let expands_to: bool = match expand_authored(&found, inputs, &scope.cli_vars) { + Ok(expanded) => expanded == warehouse, + Err(error) if expected_resolution_error(py, &error)? => false, + Err(error) => return Err(error), + }; + if expands_to { + return resolved_warehouse(py, Some(&warehouse), &found.authored.source()); + } + } + resolved_warehouse(py, Some(&warehouse), CONNECTION_WAREHOUSE_SOURCE) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(validate_cli_warehouse, module)?)?; + module.add_function(wrap_pyfunction!(find_target_group_warehouse, module)?)?; + module.add_function(wrap_pyfunction!(resolve_command_warehouse, module)?)?; + module.add_function(wrap_pyfunction!(apply_command_warehouse, module)?)?; + module.add_function(wrap_pyfunction!(describe_connection_warehouse, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs new file mode 100644 index 0000000000..60a366e763 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -0,0 +1,124 @@ +//! Cost telemetry that never changes a build result: failures become a logged warning. + +use std::path::PathBuf; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTypeMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cost_report::main::format_cost_breakdown::format_cost_breakdown; +use sqlbuild_core::cost_report::models::BreakdownOptions; + +use crate::bindings::_helpers::cost_report::records::cost_run; +use crate::bindings::_helpers::cost_report::store; + +const LOGGER_NAME: &str = "sqlbuild.cli.commands._helpers.cost.collection"; +const BUILD_BREAKDOWN_LIMIT: usize = 5; +const FALLBACK_TERMINAL_COLUMNS: usize = 100; + +/// Log the failure with its traceback, as `_LOGGER.exception` inside the handler did. +fn log_failure(py: Python<'_>, message: &str, error: &PyErr) -> PyResult<()> { + let logger: Bound<'_, PyAny> = py + .import("logging")? + .call_method1("getLogger", (LOGGER_NAME,))?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("exc_info", error.value(py))?; + logger.call_method("exception", (message,), Some(&keywords))?; + Ok(()) +} + +fn error_type(py: Python<'_>, error: &PyErr) -> PyResult { + Ok(error.get_type(py).name()?.to_string()) +} + +/// `_write_warning`: a warning line on the build's stream; its own failure is only logged. +fn warn(output_stream: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { + let written: PyResult<()> = output_stream + .call_method1("write", (format!("\nWARNING: {message}\n"),)) + .and_then(|_| output_stream.call_method0("flush")) + .map(|_| ()); + match written { + Ok(()) => Ok(()), + Err(error) => log_failure( + output_stream.py(), + "automatic run-cost warning output failed", + &error, + ), + } +} + +/// Persist a run's cost artifact; any failure is logged and reported as a warning. +#[pyfunction] +#[pyo3(signature = (*, project_dir, record, output_stream, stage))] +fn persist_cost_run( + project_dir: PathBuf, + record: &Bound<'_, PyAny>, + output_stream: &Bound<'_, PyAny>, + stage: &str, +) -> PyResult<()> { + let py: Python<'_> = record.py(); + let Err(error) = cost_run(record).and_then(|run| store::write(py, &project_dir, &run)) else { + return Ok(()); + }; + log_failure(py, &format!("automatic run-cost {stage} failed"), &error)?; + warn( + output_stream, + &format!( + "Cost telemetry failed during {stage} ({}); build result unchanged.", + error_type(py, &error)? + ), + ) +} + +fn breakdown_text(py: Python<'_>, record: &Bound<'_, PyAny>, use_color: bool) -> PyResult { + let terminal_width: usize = py + .import("shutil")? + .call_method1("get_terminal_size", ((FALLBACK_TERMINAL_COLUMNS, 24),))? + .getattr("columns")? + .extract()?; + Ok(format_cost_breakdown( + &cost_run(record)?, + BreakdownOptions { + use_color, + limit: Some(BUILD_BREAKDOWN_LIMIT), + terminal_width, + show_run_id: false, + }, + )) +} + +/// Write the build's cost breakdown; an `Exception` while rendering becomes a warning. +#[pyfunction] +#[pyo3(signature = (*, record, output_stream, use_color))] +fn write_build_cost_breakdown( + record: &Bound<'_, PyAny>, + output_stream: &Bound<'_, PyAny>, + use_color: bool, +) -> PyResult<()> { + let py: Python<'_> = record.py(); + let written: PyResult<()> = breakdown_text(py, record, use_color) + .and_then(|text| output_stream.call_method1("write", (format!("\n{text}\n"),))) + .and_then(|_| output_stream.call_method0("flush")) + .map(|_| ()); + let Err(error) = written else { + return Ok(()); + }; + if !error.is_instance_of::(py) { + return Err(error); + } + log_failure(py, "automatic run-cost output failed", &error)?; + warn( + output_stream, + &format!( + "Cost telemetry failed during output rendering ({}); build result unchanged.", + error_type(py, &error)? + ), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(persist_cost_run, module)?)?; + module.add_function(wrap_pyfunction!(write_build_cost_breakdown, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs new file mode 100644 index 0000000000..3e818f3b23 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs @@ -0,0 +1,155 @@ +//! The cost run store and cost reports as `_native` functions. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cost_report::main::cost_history_json::cost_history_json; +use sqlbuild_core::cost_report::main::cost_output_payload::cost_output_payload as output_payload; +use sqlbuild_core::cost_report::main::cost_report_json::cost_report_json; +use sqlbuild_core::cost_report::main::format_cost_breakdown::format_cost_breakdown as breakdown; +use sqlbuild_core::cost_report::main::format_cost_history::format_cost_history as history; +use sqlbuild_core::cost_report::main::select_cost_run::select_cost_run; +use sqlbuild_core::cost_report::models::{BreakdownOptions, CostRun}; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::cost_report::records::{ + cost_run, cost_runs, record_object, record_tuple, +}; +use crate::bindings::_helpers::cost_report::store; +use crate::bindings::_helpers::scopes::scope_report_objects::json_to_python; + +const BREAKDOWN_OPTIONS: (&[&str], &[&str]) = ( + &["record", "use_color"], + &["limit", "terminal_width", "show_run_id"], +); +const FALLBACK_TERMINAL_COLUMNS: usize = 100; + +fn optional_record<'py>( + py: Python<'py>, + run: Option, +) -> PyResult>> { + run.map(|run| record_object(py, &run)).transpose() +} + +/// Atomically persist one run's cost artifact and return its path. +#[pyfunction] +#[pyo3(signature = (*, project_dir, record))] +fn write_cost_run( + py: Python<'_>, + project_dir: PathBuf, + record: &Bound<'_, PyAny>, +) -> PyResult { + store::write(py, &project_dir, &cost_run(record)?) +} + +/// Read the run stored under an exact run ID, or `None`. +#[pyfunction] +#[pyo3(signature = (*, project_dir, run_id))] +fn read_cost_run<'py>( + py: Python<'py>, + project_dir: PathBuf, + run_id: &str, +) -> PyResult>> { + let run: Option = store::read(py, &project_dir, run_id)?; + optional_record(py, run) +} + +/// Every readable persisted run, newest first. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +fn list_cost_runs<'py>(py: Python<'py>, project_dir: PathBuf) -> PyResult> { + Ok(record_tuple(py, &store::list(py, &project_dir)?)?.into_any()) +} + +/// The latest finished run, or the only run whose ID starts with the selector. +#[pyfunction] +#[pyo3(signature = (*, project_dir, selector))] +fn resolve_cost_run<'py>( + py: Python<'py>, + project_dir: PathBuf, + selector: &str, +) -> PyResult>> { + let runs: Vec = store::list(py, &project_dir)?; + optional_record(py, select_cost_run(runs, selector)) +} + +/// `sqb cost --json` text of one run. +#[pyfunction] +fn format_cost_json(record: &Bound<'_, PyAny>) -> PyResult { + Ok(cost_report_json(&cost_run(record)?)) +} + +/// `sqb cost history --json` text of the shown runs. +#[pyfunction] +#[pyo3(signature = (*, records, matching_count = None))] +fn format_cost_history_json( + records: &Bound<'_, PyAny>, + matching_count: Option, +) -> PyResult { + Ok(cost_history_json(&cost_runs(records)?, matching_count)) +} + +/// The `sqb cost --json` document of one run, as a dictionary. +#[pyfunction] +#[pyo3(signature = (*, record))] +fn cost_output_payload(py: Python<'_>, record: &Bound<'_, PyAny>) -> PyResult> { + json_to_python(py, output_payload(&cost_run(record)?), false) +} + +/// The aligned human breakdown of one run. +#[pyfunction] +#[pyo3(signature = (**options))] +fn format_cost_breakdown(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "format_cost_breakdown", options, BREAKDOWN_OPTIONS)?; + let limit: Bound<'_, PyAny> = arguments.value("limit")?; + let width: Bound<'_, PyAny> = arguments.value("terminal_width")?; + let terminal_width: usize = if width.is_truthy()? { + width.extract()? + } else { + py.import("shutil")? + .call_method1("get_terminal_size", ((FALLBACK_TERMINAL_COLUMNS, 24),))? + .getattr("columns")? + .extract()? + }; + let show_run_id: bool = + !arguments.passed("show_run_id")? || arguments.value("show_run_id")?.is_truthy()?; + Ok(breakdown( + &cost_run(&arguments.value("record")?)?, + BreakdownOptions { + use_color: arguments.value("use_color")?.is_truthy()?, + limit: if limit.is_none() { + None + } else { + Some(limit.extract()?) + }, + terminal_width, + show_run_id, + }, + )) +} + +/// The aligned human history of the shown runs. +#[pyfunction] +#[pyo3(signature = (*, records, matching_count, use_color))] +fn format_cost_history( + records: &Bound<'_, PyAny>, + matching_count: usize, + use_color: bool, +) -> PyResult { + Ok(history(&cost_runs(records)?, matching_count, use_color)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_cost_run, module)?)?; + module.add_function(wrap_pyfunction!(read_cost_run, module)?)?; + module.add_function(wrap_pyfunction!(list_cost_runs, module)?)?; + module.add_function(wrap_pyfunction!(resolve_cost_run, module)?)?; + module.add_function(wrap_pyfunction!(format_cost_json, module)?)?; + module.add_function(wrap_pyfunction!(format_cost_history_json, module)?)?; + module.add_function(wrap_pyfunction!(cost_output_payload, module)?)?; + module.add_function(wrap_pyfunction!(format_cost_breakdown, module)?)?; + module.add_function(wrap_pyfunction!(format_cost_history, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs new file mode 100644 index 0000000000..242aaff1b3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod degradation; +pub(crate) mod functions; +pub(crate) mod records; +pub(crate) mod registration; +pub(crate) mod store; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs new file mode 100644 index 0000000000..4d9322f5fa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs @@ -0,0 +1,231 @@ +//! `CostRunRecord` objects read into plain cost runs, and plain runs built back into records. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_core::cost_report::errors::CostArtifactError; +use sqlbuild_core::cost_report::models::{CostResource, CostRun, CostSummary}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const COST_MODELS: &str = "sqlbuild.cost.models"; +const COST_TYPES: &str = "sqlbuild.cost.types"; +const COST_EXCEPTIONS: &str = "sqlbuild.cost.exceptions"; + +/// The `CostArtifactError` Python raised for the same artifact. +pub(crate) fn artifact_error(py: Python<'_>, error: &CostArtifactError) -> PyErr { + match imported(py, COST_EXCEPTIONS, "CostArtifactError") + .and_then(|class| class.call1((error.message.as_str(),))) + { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +fn text(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + value.getattr(field)?.extract() +} + +fn optional_text(value: &Bound<'_, PyAny>, field: &str) -> PyResult> { + value.getattr(field)?.extract() +} + +/// `str(value.)`, the text a `Decimal` field is stored as. +fn decimal(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + Ok(value.getattr(field)?.str()?.to_string()) +} + +fn integer(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + value.getattr(field)?.extract() +} + +fn timestamp(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + value.getattr(field)?.call_method0("isoformat")?.extract() +} + +fn resource(value: &Bound<'_, PyAny>) -> PyResult { + Ok(CostResource { + resource_type: text(value, "resource_type")?, + resource_name: text(value, "resource_name")?, + warehouse_name: text(value, "warehouse_name")?, + warehouse_size: text(value, "warehouse_size")?, + warehouse_type: optional_text(value, "warehouse_type")?, + cluster_number: value.getattr("cluster_number")?.extract()?, + query_count: integer(value, "query_count")?, + attributed_seconds: decimal(value, "attributed_seconds")?, + bytes_scanned: integer(value, "bytes_scanned")?, + estimated_compute_credits: decimal(value, "estimated_compute_credits")?, + estimated_usd: decimal(value, "estimated_usd")?, + }) +} + +fn summary(value: &Bound<'_, PyAny>) -> PyResult { + let mut resources: Vec = Vec::new(); + for item in value.getattr("resources")?.try_iter()? { + resources.push(resource(&item?)?); + } + let mut limitations: Vec = Vec::new(); + for item in value.getattr("limitations")?.try_iter()? { + limitations.push(item?.extract()?); + } + Ok(CostSummary { + status: value.getattr("status")?.getattr("value")?.extract()?, + usd_per_credit: decimal(value, "usd_per_credit")?, + rate_source: text(value, "rate_source")?, + resources, + query_count: integer(value, "query_count")?, + attributed_seconds: decimal(value, "attributed_seconds")?, + bytes_scanned: integer(value, "bytes_scanned")?, + estimated_compute_credits: decimal(value, "estimated_compute_credits")?, + estimated_usd: decimal(value, "estimated_usd")?, + expected_statement_count: integer(value, "expected_statement_count")?, + observed_statement_count: integer(value, "observed_statement_count")?, + missing_statement_count: integer(value, "missing_statement_count")?, + skipped_statement_count: integer(value, "skipped_statement_count")?, + source: text(value, "source")?, + method: text(value, "method")?, + limitations, + message: optional_text(value, "message")?, + }) +} + +/// A `CostRunRecord` as a plain run. +pub(crate) fn cost_run(record: &Bound<'_, PyAny>) -> PyResult { + Ok(CostRun { + run_id: text(record, "run_id")?, + adapter_name: text(record, "adapter_name")?, + target_name: optional_text(record, "target_name")?, + build_status: text(record, "build_status")?, + started_at: timestamp(record, "started_at")?, + completed_at: timestamp(record, "completed_at")?, + cost: summary(&record.getattr("cost")?)?, + version: integer(record, "version")?, + had_executable_work: record.getattr("had_executable_work")?.extract()?, + }) +} + +/// Every record of an iterable as a plain run. +pub(crate) fn cost_runs(records: &Bound<'_, PyAny>) -> PyResult> { + let mut runs: Vec = Vec::new(); + for record in records.try_iter()? { + runs.push(cost_run(&record?)?); + } + Ok(runs) +} + +/// Python constructors of the record fields. +struct Builders<'py> { + decimal: Bound<'py, PyAny>, + timestamp: Bound<'py, PyAny>, +} + +impl<'py> Builders<'py> { + fn new(py: Python<'py>) -> PyResult { + Ok(Self { + decimal: imported(py, "decimal", "Decimal")?, + timestamp: imported(py, "datetime", "datetime")?.getattr("fromisoformat")?, + }) + } + + fn decimal(&self, text: &str) -> PyResult> { + self.decimal.call1((text,)) + } +} + +fn resource_object<'py>( + builders: &Builders<'py>, + resource: &CostResource, +) -> PyResult> { + let py: Python<'py> = builders.decimal.py(); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("resource_type", &resource.resource_type)?; + fields.set_item("resource_name", &resource.resource_name)?; + fields.set_item("warehouse_name", &resource.warehouse_name)?; + fields.set_item("warehouse_size", &resource.warehouse_size)?; + fields.set_item("warehouse_type", &resource.warehouse_type)?; + fields.set_item("cluster_number", resource.cluster_number)?; + fields.set_item("query_count", resource.query_count)?; + fields.set_item( + "attributed_seconds", + builders.decimal(&resource.attributed_seconds)?, + )?; + fields.set_item("bytes_scanned", resource.bytes_scanned)?; + fields.set_item( + "estimated_compute_credits", + builders.decimal(&resource.estimated_compute_credits)?, + )?; + fields.set_item("estimated_usd", builders.decimal(&resource.estimated_usd)?)?; + imported(py, COST_MODELS, "ResourceCost")?.call((), Some(&fields)) +} + +fn summary_object<'py>( + builders: &Builders<'py>, + cost: &CostSummary, +) -> PyResult> { + let py: Python<'py> = builders.decimal.py(); + let mut resources: Vec> = Vec::with_capacity(cost.resources.len()); + for resource in &cost.resources { + resources.push(resource_object(builders, resource)?); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item( + "status", + imported(py, COST_TYPES, "CostStatus")?.call1((cost.status.as_str(),))?, + )?; + fields.set_item("usd_per_credit", builders.decimal(&cost.usd_per_credit)?)?; + fields.set_item("rate_source", &cost.rate_source)?; + fields.set_item("resources", PyTuple::new(py, resources)?)?; + fields.set_item("query_count", cost.query_count)?; + fields.set_item( + "attributed_seconds", + builders.decimal(&cost.attributed_seconds)?, + )?; + fields.set_item("bytes_scanned", cost.bytes_scanned)?; + fields.set_item( + "estimated_compute_credits", + builders.decimal(&cost.estimated_compute_credits)?, + )?; + fields.set_item("estimated_usd", builders.decimal(&cost.estimated_usd)?)?; + fields.set_item("expected_statement_count", cost.expected_statement_count)?; + fields.set_item("observed_statement_count", cost.observed_statement_count)?; + fields.set_item("missing_statement_count", cost.missing_statement_count)?; + fields.set_item("skipped_statement_count", cost.skipped_statement_count)?; + fields.set_item("source", &cost.source)?; + fields.set_item("method", &cost.method)?; + fields.set_item("limitations", PyTuple::new(py, &cost.limitations)?)?; + fields.set_item("message", &cost.message)?; + imported(py, COST_MODELS, "RunCostSummary")?.call((), Some(&fields)) +} + +/// A plain run as the `CostRunRecord` it was read from. +pub(crate) fn record_object<'py>(py: Python<'py>, run: &CostRun) -> PyResult> { + let builders: Builders<'py> = Builders::new(py)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("version", run.version)?; + fields.set_item("run_id", &run.run_id)?; + fields.set_item("adapter_name", &run.adapter_name)?; + fields.set_item("target_name", &run.target_name)?; + fields.set_item("build_status", &run.build_status)?; + fields.set_item( + "started_at", + builders.timestamp.call1((run.started_at.as_str(),))?, + )?; + fields.set_item( + "completed_at", + builders.timestamp.call1((run.completed_at.as_str(),))?, + )?; + fields.set_item("cost", summary_object(&builders, &run.cost)?)?; + fields.set_item("had_executable_work", run.had_executable_work)?; + imported(py, COST_MODELS, "CostRunRecord")?.call((), Some(&fields)) +} + +/// Records of several plain runs, as a tuple. +pub(crate) fn record_tuple<'py>( + py: Python<'py>, + runs: &[CostRun], +) -> PyResult> { + let mut records: Vec> = Vec::with_capacity(runs.len()); + for run in runs { + records.push(record_object(py, run)?); + } + PyTuple::new(py, records) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs new file mode 100644 index 0000000000..7dbf137270 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs @@ -0,0 +1,10 @@ +//! Register the cost run store and the cost reports. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::cost_report::{degradation, functions}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + degradation::register(module)?; + functions::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs new file mode 100644 index 0000000000..bf599f24ad --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs @@ -0,0 +1,126 @@ +//! Project-local cost run artifacts under `target/executions//run.json`. + +use std::fs; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyResult, Python}; +use sqlbuild_core::cost_report::errors::CostArtifactError; +use sqlbuild_core::cost_report::main::check_cost_run_id::check_cost_run_id; +use sqlbuild_core::cost_report::main::cost_artifact_text::cost_artifact_text; +use sqlbuild_core::cost_report::main::order_cost_runs::order_cost_runs; +use sqlbuild_core::cost_report::main::read_cost_artifact::read_cost_artifact; +use sqlbuild_core::cost_report::models::CostRun; + +use crate::bindings::_helpers::cost_report::records::artifact_error; +use crate::bindings::_helpers::result_output::integration_values::json_value; + +const ARTIFACT_NAME: &str = "run.json"; +const TEMPORARY_NAME: &str = ".run.json.tmp"; + +fn invalid(message: &str) -> CostArtifactError { + CostArtifactError { + message: message.to_owned(), + } +} + +/// The directory every run's artifact lives under. +fn executions_root(project_dir: &Path) -> PathBuf { + project_dir.join("target").join("executions") +} + +/// `_run_directory`: the run's directory, which must stay under the executions root. +fn run_directory(project_dir: &Path, run_id: &str) -> Result { + check_cost_run_id(run_id)?; + let root: PathBuf = executions_root(project_dir); + let directory: PathBuf = root.join(run_id); + if let (Ok(resolved), Ok(resolved_root)) = (directory.canonicalize(), root.canonicalize()) + && !resolved.starts_with(resolved_root) + { + return Err(invalid( + "cost run path must remain under the project execution directory", + )); + } + Ok(directory) +} + +/// The run an artifact file describes; I/O failures stay Python `OSError`s. +fn load(py: Python<'_>, path: &Path) -> PyResult> { + let bytes: Vec = fs::read(path).map_err(PyErr::from)?; + let Ok(text) = String::from_utf8(bytes) else { + return Ok(Err(invalid("cost run artifact is not UTF-8 text"))); + }; + let json: Bound<'_, PyModule> = py.import("json")?; + let payload: Bound<'_, PyAny> = match json.call_method1("loads", (text,)) { + Ok(payload) => payload, + Err(error) if error.is_instance(py, &json.getattr("JSONDecodeError")?) => { + return Ok(Err(invalid("cost run artifact is not valid JSON"))); + } + Err(error) => return Err(error), + }; + let Some(payload) = json_value(&payload)? else { + return Ok(Err(invalid("cost run artifact must be an object"))); + }; + Ok(read_cost_artifact(&payload)) +} + +/// Raise the artifact failure Python reported for the same file. +fn checked(py: Python<'_>, result: Result) -> PyResult { + result.map_err(|error| artifact_error(py, &error)) +} + +/// `write_cost_run`: atomically replace the run's artifact and return its path. +pub(crate) fn write(py: Python<'_>, project_dir: &Path, run: &CostRun) -> PyResult { + let directory: PathBuf = checked(py, run_directory(project_dir, &run.run_id))?; + fs::create_dir_all(&directory).map_err(PyErr::from)?; + let destination: PathBuf = directory.join(ARTIFACT_NAME); + let temporary: PathBuf = directory.join(TEMPORARY_NAME); + fs::write(&temporary, cost_artifact_text(run)).map_err(PyErr::from)?; + fs::rename(&temporary, &destination).map_err(PyErr::from)?; + Ok(destination) +} + +/// `read_cost_run`: the run stored under the ID, `None` when it has no artifact. +pub(crate) fn read(py: Python<'_>, project_dir: &Path, run_id: &str) -> PyResult> { + let path: PathBuf = checked(py, run_directory(project_dir, run_id))?.join(ARTIFACT_NAME); + if !path.is_file() { + return Ok(None); + } + let run: CostRun = checked(py, load(py, &path)?)?; + if run.run_id != run_id { + return Err(artifact_error( + py, + &invalid("cost run artifact run ID does not match its directory"), + )); + } + Ok(Some(run)) +} + +fn listed_run(py: Python<'_>, root: &Path, directory: &Path) -> PyResult> { + let path: PathBuf = directory.join(ARTIFACT_NAME); + let (Ok(resolved), Ok(resolved_root)) = (path.canonicalize(), root.canonicalize()) else { + return Ok(None); + }; + if !resolved.starts_with(resolved_root) || !path.is_file() { + return Ok(None); + } + let Ok(Ok(run)) = load(py, &path) else { + return Ok(None); + }; + let name: Option<&str> = directory.file_name().and_then(|name| name.to_str()); + Ok((name == Some(run.run_id.as_str())).then_some(run)) +} + +/// `list_cost_runs`: every readable run, newest first; unreadable artifacts are skipped. +pub(crate) fn list(py: Python<'_>, project_dir: &Path) -> PyResult> { + let root: PathBuf = executions_root(project_dir); + let Ok(entries) = fs::read_dir(&root) else { + return Ok(Vec::new()); + }; + let mut runs: Vec = Vec::new(); + for entry in entries.flatten() { + if let Some(run) = listed_run(py, &root, &entry.path())? { + runs.push(run); + } + } + Ok(order_cost_runs(runs)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs new file mode 100644 index 0000000000..80c115906e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs @@ -0,0 +1,278 @@ +//! One call's cursor values read from and built as `sqlbuild.cursor_algebra.models` objects. + +use std::cell::RefCell; + +use pyo3::IntoPyObject; +use pyo3::exceptions::{PyOverflowError, PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; +use sqlbuild_cursor::algebra::errors::CursorError; +use sqlbuild_cursor::algebra::models::{ + Grain, Interval, LocalDateTime, Scalar, ScalarValue, Timestamp, ZoneOffset, +}; +use sqlbuild_cursor::algebra::types::ZoneRules; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::values::{ + DATETIME_MODULE, civil_date, local_date_time, naive_date_time, plain_date, replaced, + timedelta_micros, +}; + +pub(crate) const MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.cursor_algebra.exceptions"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const GRAIN_NAMES: [(&str, Grain); 6] = [ + ("second", Grain::Second), + ("minute", Grain::Minute), + ("hour", Grain::Hour), + ("day", Grain::Day), + ("month", Grain::Month), + ("year", Grain::Year), +]; + +/// `CursorAlgebraError(message)`. +pub(crate) fn algebra_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, "CursorAlgebraError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// The cursor values one native call reads, so unchanged inputs come back as the same objects. +pub(crate) struct CursorSession<'py> { + py: Python<'py>, + timestamp_class: Bound<'py, PyAny>, + date_class: Bound<'py, PyAny>, + integer_class: Bound<'py, PyAny>, + /// Each input value object and the `date`, `datetime` or `int` it holds. + inputs: RefCell, Bound<'py, PyAny>)>>, + /// The Python error a zone lookup raised, re-raised in place of the algebra's error. + failure: RefCell>, +} + +impl<'py> CursorSession<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + let models = py.import(MODELS_MODULE)?; + Ok(Self { + py, + timestamp_class: models.getattr("TimestampValue")?, + date_class: models.getattr("DateValue")?, + integer_class: models.getattr("IntegerValue")?, + inputs: RefCell::new(Vec::new()), + failure: RefCell::new(None), + }) + } + + pub(crate) fn py(&self) -> Python<'py> { + self.py + } + + /// Whether `value` is one of the three typed cursor values. + pub(crate) fn scalar_typed(&self, value: &Bound<'py, PyAny>) -> PyResult { + Ok(value.is_instance(&self.timestamp_class)? + || value.is_instance(&self.date_class)? + || value.is_instance(&self.integer_class)?) + } + + fn timestamp(&self, inner: &Bound<'py, PyAny>, handle: usize) -> PyResult { + let tzinfo: Bound<'py, PyAny> = inner.getattr("tzinfo")?; + let offset: ZoneOffset = if tzinfo.is_none() { + ZoneOffset::Naive + } else if tzinfo + .get_type() + .is(&imported(self.py, DATETIME_MODULE, "timezone")?) + { + ZoneOffset::Fixed(timedelta_micros( + &tzinfo.call_method1("utcoffset", (self.py.None(),))?, + )?) + } else { + ZoneOffset::External(handle) + }; + Ok(Timestamp { + local: local_date_time(inner)?, + fold: inner.getattr("fold")?.extract::()? != 0, + offset, + template: Some(handle), + source_text: None, + }) + } + + /// A typed cursor value object as a scalar remembering the object. + pub(crate) fn scalar(&self, value: &Bound<'py, PyAny>) -> PyResult { + let inner: Bound<'py, PyAny> = value.getattr("value")?; + let handle: usize = self.inputs.borrow().len(); + let typed: ScalarValue = if value.is_instance(&self.integer_class)? { + ScalarValue::Integer(inner.extract()?) + } else if value.is_instance(&self.date_class)? { + ScalarValue::Date(civil_date(&inner)?) + } else { + let mut timestamp: Timestamp = self.timestamp(&inner, handle)?; + let source_text: Bound<'py, PyAny> = value.getattr("source_text")?; + timestamp.source_text = (!source_text.is_none()) + .then(|| source_text.extract()) + .transpose()?; + ScalarValue::Timestamp(timestamp) + }; + self.inputs.borrow_mut().push((value.clone(), inner)); + Ok(Scalar { + value: typed, + handle: Some(handle), + }) + } + + /// `CursorGrain | None` as a grain; an unknown name fails as `CursorGrain(name)` does. + pub(crate) fn grain(&self, grain: &Bound<'py, PyAny>) -> PyResult> { + if grain.is_none() { + return Ok(None); + } + if let Ok(name) = grain.downcast::() { + let name: String = name.to_str()?.to_owned(); + if let Some((_, known)) = GRAIN_NAMES.iter().find(|(known, _)| *known == name) { + return Ok(Some(*known)); + } + } + imported(self.py, PLANNER_TYPES_MODULE, "CursorGrain")?.call1((grain,))?; + Err(PyTypeError::new_err("cursor grain must be a CursorGrain")) + } + + /// An `AlignedInterval` as an interval. + pub(crate) fn interval(&self, interval: &Bound<'py, PyAny>) -> PyResult { + Ok(Interval { + start: self.scalar(&interval.getattr("start")?)?, + end: self.scalar(&interval.getattr("end")?)?, + grain: self.grain(&interval.getattr("grain")?)?, + }) + } + + fn input(&self, handle: usize, inner: bool) -> Bound<'py, PyAny> { + let inputs = self.inputs.borrow(); + let (value, held) = &inputs[handle]; + if inner { held.clone() } else { value.clone() } + } + + /// The `date`, `datetime` or `int` input value `handle` holds. + pub(crate) fn held(&self, handle: usize) -> Bound<'py, PyAny> { + self.input(handle, true) + } + + /// The `datetime` holding `local` in the zone and type of input `template`. + fn date_time(&self, timestamp: &Timestamp) -> PyResult> { + match timestamp.template { + Some(template) => replaced( + &self.input(template, true), + ×tamp.local, + timestamp.fold, + ), + None => naive_date_time(self.py, ×tamp.local, timestamp.fold), + } + } + + /// The input object for an unchanged value, else a new typed cursor value. + pub(crate) fn python_scalar(&self, scalar: &Scalar) -> PyResult> { + if let Some(handle) = scalar.handle { + return Ok(self.input(handle, false)); + } + let (class, value): (&Bound<'py, PyAny>, Bound<'py, PyAny>) = match &scalar.value { + ScalarValue::Integer(number) => ( + &self.integer_class, + number.into_pyobject(self.py)?.into_any(), + ), + ScalarValue::Date(date) => (&self.date_class, plain_date(self.py, date)?), + ScalarValue::Timestamp(timestamp) => { + (&self.timestamp_class, self.date_time(timestamp)?) + } + }; + call_with(class, &[("value", value)]) + } + + /// A grain as its `CursorGrain` member. + pub(crate) fn python_grain(&self, grain: Option) -> PyResult> { + let Some(grain) = grain else { + return Ok(self.py.None().into_bound(self.py)); + }; + let name: &str = GRAIN_NAMES + .iter() + .find(|(_, known)| *known == grain) + .map_or("", |(name, _)| name); + imported(self.py, PLANNER_TYPES_MODULE, "CursorGrain")?.call1((name,)) + } + + /// A new `AlignedInterval`, validated as its constructor validates. + pub(crate) fn python_interval(&self, interval: &Interval) -> PyResult> { + call_with( + &imported(self.py, MODELS_MODULE, "AlignedInterval")?, + &[ + ("start", self.python_scalar(&interval.start)?), + ("end", self.python_scalar(&interval.end)?), + ("grain", self.python_grain(interval.grain)?), + ], + ) + } + + /// The Python exception for a failed cursor operation. + pub(crate) fn error(&self, error: CursorError) -> PyErr { + if let Some(failure) = self.failure.borrow_mut().take() { + return failure; + } + match error { + CursorError::Algebra(message) => algebra_error(self.py, &message), + CursorError::Overflow(message) => PyOverflowError::new_err(message), + CursorError::Value(message) => PyValueError::new_err(message), + CursorError::Type(message) => PyTypeError::new_err(message), + } + } + + /// `result`, with a cursor error raised as its Python exception. + pub(crate) fn checked(&self, result: Result) -> PyResult { + result.map_err(|error| self.error(error)) + } + + fn zone_call( + &self, + template: usize, + local: &LocalDateTime, + fold: bool, + ) -> PyResult> { + replaced(&self.input(template, true), local, fold) + } + + fn remember(&self, error: PyErr) -> CursorError { + let message: String = error.to_string(); + *self.failure.borrow_mut() = Some(error); + CursorError::Algebra(message) + } +} + +impl ZoneRules for CursorSession<'_> { + fn to_utc( + &self, + template: usize, + local: &LocalDateTime, + fold: bool, + ) -> Result { + let converted = self.zone_call(template, local, fold).and_then(|value| { + let utc: Bound<'_, PyAny> = imported(self.py, DATETIME_MODULE, "UTC")?; + local_date_time(&value.call_method1("astimezone", (utc,))?) + }); + converted.map_err(|error| self.remember(error)) + } + + fn utc_offset( + &self, + template: usize, + local: &LocalDateTime, + fold: bool, + ) -> Result, CursorError> { + let offset = self.zone_call(template, local, fold).and_then(|value| { + let delta: Bound<'_, PyAny> = value.call_method0("utcoffset")?; + if delta.is_none() { + return Ok(None); + } + timedelta_micros(&delta).map(Some) + }); + offset.map_err(|error| self.remember(error)) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs new file mode 100644 index 0000000000..a31d7399ca --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs @@ -0,0 +1,96 @@ +//! Calendar durations parsed from text and applied to dates and timestamps. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyInt, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_cursor::algebra::main::shift_by_duration::shift_by_duration; +use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan, Moment}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::values::{ + DATETIME_MODULE, civil_date, local_date_time, replaced, +}; + +const MONTHS_PER_YEAR: i64 = 12; + +/// `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. +#[pyfunction] +#[pyo3(signature = (value))] +fn parse_cursor_duration<'py>( + py: Python<'py>, + value: &str, +) -> PyResult>> { + let Some(digits): Option = parse_duration(value) else { + return Ok(None); + }; + let int = py.get_type::(); + let parts: Vec> = [ + digits.years, + digits.months, + digits.days, + digits.hours, + digits.minutes, + digits.seconds, + ] + .iter() + .map(|text| int.call1((text.as_str(),))) + .collect::>()?; + PyTuple::new(py, parts).map(Some) +} + +fn duration_span(duration: &Bound<'_, PyAny>) -> PyResult { + let field = |name: &str| -> PyResult { duration.getattr(name)?.extract() }; + Ok(DurationSpan { + months: field("years")? * MONTHS_PER_YEAR + field("months")?, + days: field("days")?, + hours: field("hours")?, + minutes: field("minutes")?, + seconds: field("seconds")?, + }) +} + +/// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. +#[pyfunction] +#[pyo3(signature = (*, moment, duration, direction))] +fn shift_by_cursor_duration<'py>( + moment: &Bound<'py, PyAny>, + duration: &Bound<'py, PyAny>, + direction: i64, +) -> PyResult> { + let py: Python<'py> = moment.py(); + let session: CursorSession<'py> = CursorSession::new(py)?; + let is_timestamp: bool = moment.is_instance(&imported(py, DATETIME_MODULE, "datetime")?)?; + let typed: Moment = if is_timestamp { + Moment::Timestamp { + local: local_date_time(moment)?, + fold: moment.getattr("fold")?.extract::()? != 0, + } + } else { + Moment::Date(civil_date(moment)?) + }; + let shifted: Moment = session.checked(shift_by_duration( + typed, + &duration_span(duration)?, + direction, + ))?; + match shifted { + Moment::Timestamp { local, fold } => replaced(moment, &local, fold), + Moment::Date(date) => call_with( + &moment.getattr("replace")?, + &[ + ("year", date.year.into_pyobject(py)?.into_any()), + ("month", date.month.into_pyobject(py)?.into_any()), + ("day", date.day.into_pyobject(py)?.into_any()), + ], + ), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_cursor_duration, module)?)?; + module.add_function(wrap_pyfunction!(shift_by_cursor_duration, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs new file mode 100644 index 0000000000..3831b2b702 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs @@ -0,0 +1,131 @@ +//! Building, intersecting, merging, splitting and capping half-open cursor intervals. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PySlice, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::main::cap_from_end::cap_from_end; +use sqlbuild_cursor::algebra::main::cap_from_start::cap_from_start; +use sqlbuild_cursor::algebra::main::clamp::clamp; +use sqlbuild_cursor::algebra::main::merge::merge; +use sqlbuild_cursor::algebra::main::observed_partition::observed_partition; +use sqlbuild_cursor::algebra::main::split_into_batches::split_into_batches; +use sqlbuild_cursor::algebra::main::validate_interval::validate_interval; +use sqlbuild_cursor::algebra::models::{Interval, Scalar}; + +use crate::bindings::_helpers::cursor::conversion::CursorSession; + +fn python_intervals<'py>( + session: &CursorSession<'py>, + intervals: &[Interval], +) -> PyResult> { + let built: Vec> = intervals + .iter() + .map(|interval| session.python_interval(interval)) + .collect::>()?; + PyTuple::new(session.py(), built) +} + +/// Raise `CursorAlgebraError` unless the ends form a valid aligned interval. +#[pyfunction(name = "validate_cursor_interval")] +#[pyo3(signature = (*, start, end, grain))] +fn interval_validation( + start: &Bound<'_, PyAny>, + end: &Bound<'_, PyAny>, + grain: &Bound<'_, PyAny>, +) -> PyResult<()> { + let session: CursorSession<'_> = CursorSession::new(start.py())?; + let interval = Interval { + start: session.scalar(start)?, + end: session.scalar(end)?, + grain: session.grain(grain)?, + }; + session.checked(validate_interval(&interval, &session)) +} + +/// The canonical partition containing an observed physical value. +#[pyfunction(name = "cursor_observed_partition")] +#[pyo3(signature = (*, value, grain))] +fn partition<'py>( + value: &Bound<'py, PyAny>, + grain: &Bound<'py, PyAny>, +) -> PyResult> { + let session: CursorSession<'py> = CursorSession::new(value.py())?; + let scalar: Scalar = session.scalar(value)?; + let interval: Interval = session.checked(observed_partition(&scalar, session.grain(grain)?))?; + session.python_interval(&interval) +} + +/// The intersection of two compatible half-open intervals, or `None`. +#[pyfunction(name = "cursor_clamp")] +#[pyo3(signature = (*, interval, bounds))] +fn clamp_intervals<'py>( + interval: &Bound<'py, PyAny>, + bounds: &Bound<'py, PyAny>, +) -> PyResult>> { + let session: CursorSession<'py> = CursorSession::new(interval.py())?; + let (interval, bounds): (Interval, Interval) = + (session.interval(interval)?, session.interval(bounds)?); + session + .checked(clamp(&interval, &bounds, &session))? + .map(|clamped| session.python_interval(&clamped)) + .transpose() +} + +/// Overlapping or adjacent canonical intervals merged. +#[pyfunction(name = "cursor_merge")] +#[pyo3(signature = (*, intervals))] +fn merge_intervals<'py>(intervals: &Bound<'py, PyAny>) -> PyResult> { + let session: CursorSession<'py> = CursorSession::new(intervals.py())?; + let typed: Vec = intervals + .try_iter()? + .map(|interval| session.interval(&interval?)) + .collect::>()?; + let merged: Vec = session.checked(merge(&typed, &session))?; + python_intervals(&session, &merged) +} + +/// An interval split into batches of a positive number of grain units. +#[pyfunction(name = "cursor_split_into_batches")] +#[pyo3(signature = (*, interval, step))] +fn split<'py>(interval: &Bound<'py, PyAny>, step: i64) -> PyResult> { + let session: CursorSession<'py> = CursorSession::new(interval.py())?; + let typed: Interval = session.interval(interval)?; + let batches: Vec = session.checked(split_into_batches(&typed, step, &session))?; + python_intervals(&session, &batches) +} + +fn sliced<'py>( + intervals: &Bound<'py, PyAny>, + range: std::ops::Range, +) -> PyResult> { + let slice = PySlice::new(intervals.py(), range.start as isize, range.end as isize, 1); + intervals.get_item(slice) +} + +/// At most `count` intervals from the start. +#[pyfunction(name = "cursor_cap_from_start")] +#[pyo3(signature = (*, intervals, count))] +fn cap_start<'py>(intervals: &Bound<'py, PyAny>, count: i64) -> PyResult> { + sliced(intervals, cap_from_start(intervals.len()?, count)) +} + +/// At most `count` intervals from the end; an empty tuple when `count` is not positive. +#[pyfunction(name = "cursor_cap_from_end")] +#[pyo3(signature = (*, intervals, count))] +fn cap_end<'py>(intervals: &Bound<'py, PyAny>, count: i64) -> PyResult> { + if count <= 0 { + return Ok(PyTuple::empty(intervals.py()).into_any()); + } + sliced(intervals, cap_from_end(intervals.len()?, count)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(interval_validation, module)?)?; + module.add_function(wrap_pyfunction!(partition, module)?)?; + module.add_function(wrap_pyfunction!(clamp_intervals, module)?)?; + module.add_function(wrap_pyfunction!(merge_intervals, module)?)?; + module.add_function(wrap_pyfunction!(split, module)?)?; + module.add_function(wrap_pyfunction!(cap_start, module)?)?; + module.add_function(wrap_pyfunction!(cap_end, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs new file mode 100644 index 0000000000..dec9f2ffdf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod conversion; +pub(crate) mod durations; +pub(crate) mod intervals; +pub(crate) mod registration; +pub(crate) mod scalars; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/registration.rs new file mode 100644 index 0000000000..ed10c4959d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/registration.rs @@ -0,0 +1,12 @@ +//! Register the native cursor algebra functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::cursor::{durations, intervals, scalars}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + scalars::register(module)?; + intervals::register(module)?; + durations::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs new file mode 100644 index 0000000000..259fa34a09 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs @@ -0,0 +1,416 @@ +//! Parsing, rendering, comparing and stepping typed cursor values. + +use std::cmp::Ordering; + +use pyo3::IntoPyObject; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::errors::CursorError; +use sqlbuild_cursor::algebra::main::compare::compare; +use sqlbuild_cursor::algebra::main::cursor_sort_key::cursor_sort_key as sort_key; +use sqlbuild_cursor::algebra::main::exclusive_end_from_observed_max::exclusive_end_from_observed_max; +use sqlbuild_cursor::algebra::main::exclusive_to_inclusive::exclusive_to_inclusive; +use sqlbuild_cursor::algebra::main::extreme_position::extreme_position; +use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; +use sqlbuild_cursor::algebra::main::inclusive_to_exclusive::inclusive_to_exclusive; +use sqlbuild_cursor::algebra::main::next_boundary::next_boundary; +use sqlbuild_cursor::algebra::main::normalize_timestamp_text::normalize_timestamp_text; +use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; +use sqlbuild_cursor::algebra::main::render::render; +use sqlbuild_cursor::algebra::models::{Extreme, Grain, Scalar, ScalarValue, SortKey, ZoneOffset}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::conversion::{CursorSession, MODELS_MODULE, algebra_error}; +use crate::bindings::_helpers::cursor::values::{DATETIME_MODULE, naive_date_time}; + +const DECIMAL_MODULE: &str = "decimal"; +const TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; +const INTEGER_CURSOR: &str = "integer"; +const UTC_OFFSET: i64 = 0; +const TIMESTAMP_CURSOR: &str = "timestamp"; + +/// `CursorAlgebraError(message)` raised from `cause`. +fn chained(py: Python<'_>, message: String, cause: PyErr) -> PyErr { + let error: PyErr = algebra_error(py, &message); + error.set_cause(py, Some(cause)); + error +} + +/// `IntegerValue` parsed from `str(raw)` the way `int(Decimal(str(raw)))` reads it. +fn parse_integer<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = raw.py(); + let text: String = raw.str()?.to_str()?.to_owned(); + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "IntegerValue")?; + if let Some(value) = parse_integer_text(&text) { + return call_with(&class, &[("value", value.into_pyobject(py)?.into_any())]); + } + let parsed: PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = + imported(py, DECIMAL_MODULE, "Decimal")? + .call1((text.as_str(),)) + .and_then(|decimal| { + let integer: Bound<'py, PyAny> = + py.get_type::().call1((&decimal,))?; + Ok((decimal, integer)) + }); + let (decimal, integer) = match parsed { + Ok(values) => values, + Err(error) if integer_failure(py, &error)? => { + return Err(chained( + py, + format!("invalid integer cursor value: {text}"), + error, + )); + } + Err(error) => return Err(error), + }; + if decimal.ne(&integer)? { + return Err(algebra_error( + py, + &format!("non-integral integer cursor value: {text}"), + )); + } + call_with(&class, &[("value", integer)]) +} + +/// Whether `int(Decimal(...))` failed with `InvalidOperation`, `ValueError` or `OverflowError`. +fn integer_failure(py: Python<'_>, error: &PyErr) -> PyResult { + let invalid: Bound<'_, PyAny> = imported(py, DECIMAL_MODULE, "InvalidOperation")?; + Ok(error.is_instance(py, &invalid) + || error.is_instance_of::(py) + || error.is_instance_of::(py)) +} + +/// `value.astimezone(UTC)` for an aware `datetime`, else `value`. +fn utc_normalized<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.getattr("tzinfo")?.is_none() { + return Ok(value.clone()); + } + value.call_method1( + "astimezone", + (imported(value.py(), DATETIME_MODULE, "UTC")?,), + ) +} + +/// A `DateValue` or `TimestampValue` parsed from warehouse timestamp text. +fn parse_timestamp_text<'py>(raw: &Bound<'py, PyAny>, text: &str) -> PyResult> { + let py: Python<'py> = raw.py(); + let normalized: String = normalize_timestamp_text(text); + let as_date = imported(py, DATETIME_MODULE, "date")? + .call_method1("fromisoformat", (normalized.as_str(),)); + match as_date { + Ok(date) => { + return call_with( + &imported(py, MODELS_MODULE, "DateValue")?, + &[("value", date)], + ); + } + Err(error) if error.is_instance_of::(py) => {} + Err(error) => return Err(error), + } + let parsed = imported(py, DATETIME_MODULE, "datetime")? + .call_method1("fromisoformat", (normalized.as_str(),)) + .and_then(|value| utc_normalized(&value)); + match parsed { + Ok(value) => call_with( + &imported(py, MODELS_MODULE, "TimestampValue")?, + &[ + ("value", value), + ("source_text", PyString::new(py, &normalized).into_any()), + ], + ), + Err(error) if error.is_instance_of::(py) => Err(chained( + py, + format!("invalid timestamp cursor value: {}", raw.str()?), + error, + )), + Err(error) => Err(error), + } +} + +/// One cursor value parsed from its boundary representation, keeping date-ness and offsets. +pub(crate) fn parse_value<'py>( + raw: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = raw.py(); + if cursor_type.eq(INTEGER_CURSOR)? { + return parse_integer(raw); + } + if !cursor_type.eq(TIMESTAMP_CURSOR)? { + return Err(algebra_error( + py, + &format!("unsupported cursor type: {}", cursor_type.str()?), + )); + } + if raw.is_instance(&imported(py, DATETIME_MODULE, "datetime")?)? { + let value: Bound<'py, PyAny> = utc_normalized(raw)?; + return call_with( + &imported(py, MODELS_MODULE, "TimestampValue")?, + &[("value", value)], + ); + } + if raw.is_instance(&imported(py, DATETIME_MODULE, "date")?)? { + return call_with( + &imported(py, MODELS_MODULE, "DateValue")?, + &[("value", raw.clone())], + ); + } + let Ok(text) = raw.downcast::() else { + return Err(algebra_error( + py, + &format!("invalid timestamp cursor value: {}", raw.str()?), + )); + }; + parse_timestamp_text(raw, text.to_str()?) +} + +/// `raw` unchanged when it is already a typed value, else parsed. +fn as_scalar<'py>( + session: &CursorSession<'py>, + raw: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult { + if session.scalar_typed(raw)? { + return session.scalar(raw); + } + session.scalar(&parse_value(raw, cursor_type)?) +} + +/// One cursor scalar, preserving date-ness and timestamp offsets. +#[pyfunction(name = "cursor_parse")] +#[pyo3(signature = (*, raw, cursor_type))] +fn parse<'py>( + raw: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + parse_value(raw, cursor_type) +} + +/// Discovery values that may be dates or timestamps, or `None` when they do not parse. +#[pyfunction(name = "cursor_try_parse")] +#[pyo3(signature = (*, raw, cursor_type))] +fn try_parse<'py>( + raw: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = raw.py(); + match parse_value(raw, cursor_type) { + Ok(value) => Ok(Some(value)), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + Ok(None) + } + Err(error) => Err(error), + } +} + +/// A scalar in the existing cursor output format. +#[pyfunction(name = "cursor_render")] +#[pyo3(signature = (*, value))] +pub(crate) fn render_value(value: &Bound<'_, PyAny>) -> PyResult { + let session: CursorSession<'_> = CursorSession::new(value.py())?; + let scalar: Scalar = session.scalar(value)?; + session.checked(render(&scalar, &session)) +} + +/// A sentinel's token, or a scalar rendered. +#[pyfunction(name = "cursor_sentinel_to_token")] +#[pyo3(signature = (*, sentinel))] +pub(crate) fn sentinel_to_token(sentinel: &Bound<'_, PyAny>) -> PyResult { + if sentinel.is_instance(&imported(sentinel.py(), TYPES_MODULE, "BoundSentinel")?)? { + return sentinel.getattr("value")?.extract(); + } + render_value(sentinel) +} + +/// The sentinel a SQL token names, if it names one. +#[pyfunction(name = "cursor_sentinel_from_token")] +#[pyo3(signature = (*, token))] +fn sentinel_from_token<'py>(token: &Bound<'py, PyAny>) -> PyResult>> { + let py: Python<'py> = token.py(); + match imported(py, TYPES_MODULE, "BoundSentinel")?.call1((token,)) { + Ok(sentinel) => Ok(Some(sentinel)), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } +} + +/// `-1`, `0` or `1` as `left` orders against `right`. +#[pyfunction(name = "cursor_compare")] +#[pyo3(signature = (*, left, right))] +fn compare_values(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { + let session: CursorSession<'_> = CursorSession::new(left.py())?; + let (left, right): (Scalar, Scalar) = (session.scalar(left)?, session.scalar(right)?); + Ok(match session.checked(compare(&left, &right, &session))? { + Ordering::Less => -1, + Ordering::Equal => 0, + Ordering::Greater => 1, + }) +} + +/// The key as `int` or naive `datetime`; converting from UTC itself keeps the fold. +fn python_key<'py>(py: Python<'py>, key: &SortKey, fold: bool) -> PyResult> { + match key { + SortKey::Integer(number) => Ok(number.into_pyobject(py)?.into_any()), + SortKey::Moment(local) => naive_date_time(py, local, fold), + } +} + +/// The canonical sortable key of one raw cursor bound. +#[pyfunction(name = "cursor_sort_key")] +#[pyo3(signature = (*, raw, cursor_type))] +fn sortable_key<'py>( + raw: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + let session: CursorSession<'py> = CursorSession::new(raw.py())?; + let scalar: Scalar = as_scalar(&session, raw, cursor_type)?; + let key: SortKey = session.checked(sort_key(&scalar, &session))?; + let ScalarValue::Timestamp(timestamp) = &scalar.value else { + return python_key(raw.py(), &key, false); + }; + match (timestamp.offset, timestamp.template) { + (ZoneOffset::Naive, Some(template)) => Ok(session.held(template)), + (ZoneOffset::Fixed(UTC_OFFSET), _) => python_key(raw.py(), &key, timestamp.fold), + _ => python_key(raw.py(), &key, false), + } +} + +fn extreme_bound<'py>( + values: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, + extreme: Extreme, +) -> PyResult> { + let session: CursorSession<'py> = CursorSession::new(values.py())?; + let raws: Vec> = values.try_iter()?.collect::>()?; + let mut keys: Vec = Vec::with_capacity(raws.len()); + for raw in &raws { + let scalar: Scalar = as_scalar(&session, raw, cursor_type)?; + let key: SortKey = session.checked(sort_key(&scalar, &session))?; + let unorderable: bool = keys + .first() + .is_some_and(|first| std::mem::discriminant(first) != std::mem::discriminant(&key)); + keys.push(key); + if unorderable { + break; + } + } + let position: usize = session.checked(extreme_position(&keys, extreme))?; + Ok(raws[position].clone()) +} + +/// The numerically or temporally largest raw bound. +#[pyfunction(name = "cursor_max_bound")] +#[pyo3(signature = (*, values, cursor_type))] +fn max_bound<'py>( + values: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + extreme_bound(values, cursor_type, Extreme::Max) +} + +/// The numerically or temporally smallest raw bound. +#[pyfunction(name = "cursor_min_bound")] +#[pyo3(signature = (*, values, cursor_type))] +fn min_bound<'py>( + values: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + extreme_bound(values, cursor_type, Extreme::Min) +} + +/// A scalar stepped by one grain-dependent operation. +fn stepped<'py>( + value: &Bound<'py, PyAny>, + grain: &Bound<'py, PyAny>, + step: impl FnOnce(&Scalar, Option) -> Result, +) -> PyResult> { + let session: CursorSession<'py> = CursorSession::new(value.py())?; + let scalar: Scalar = session.scalar(value)?; + let result: Scalar = session.checked(step(&scalar, session.grain(grain)?))?; + session.python_scalar(&result) +} + +/// The aligned boundary at or before a cursor value. +#[pyfunction(name = "cursor_floor_to_grain")] +#[pyo3(signature = (*, value, grain))] +fn floor_value<'py>( + value: &Bound<'py, PyAny>, + grain: &Bound<'py, PyAny>, +) -> PyResult> { + stepped(value, grain, |scalar, grain| { + Ok(floor_to_grain(scalar, grain)) + }) +} + +/// The boundary one grain after an aligned cursor value. +#[pyfunction(name = "cursor_next_boundary")] +#[pyo3(signature = (*, value, grain))] +fn next_value<'py>( + value: &Bound<'py, PyAny>, + grain: &Bound<'py, PyAny>, +) -> PyResult> { + stepped(value, grain, next_boundary) +} + +/// An operator-declared inclusive bound advanced by one cursor unit. +#[pyfunction(name = "cursor_inclusive_to_exclusive")] +#[pyo3(signature = (*, value, grain))] +fn to_exclusive<'py>( + value: &Bound<'py, PyAny>, + grain: &Bound<'py, PyAny>, +) -> PyResult> { + stepped(value, grain, inclusive_to_exclusive) +} + +/// An exclusive bound moved back by one cursor unit. +#[pyfunction(name = "cursor_exclusive_to_inclusive")] +#[pyo3(signature = (*, value, grain))] +fn to_inclusive<'py>( + value: &Bound<'py, PyAny>, + grain: &Bound<'py, PyAny>, +) -> PyResult> { + stepped(value, grain, exclusive_to_inclusive) +} + +/// The exclusive end of the partition holding an observed maximum. +#[pyfunction(name = "cursor_exclusive_end_from_observed_max")] +#[pyo3(signature = (*, value, grain))] +fn observed_end<'py>( + value: &Bound<'py, PyAny>, + grain: &Bound<'py, PyAny>, +) -> PyResult> { + let session: CursorSession<'py> = CursorSession::new(value.py())?; + let scalar: Scalar = session.scalar(value)?; + let end: Scalar = session.checked(exclusive_end_from_observed_max( + &scalar, + session.grain(grain)?, + &session, + ))?; + session.python_scalar(&end) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse, module)?)?; + module.add_function(wrap_pyfunction!(try_parse, module)?)?; + module.add_function(wrap_pyfunction!(render_value, module)?)?; + module.add_function(wrap_pyfunction!(sentinel_to_token, module)?)?; + module.add_function(wrap_pyfunction!(sentinel_from_token, module)?)?; + module.add_function(wrap_pyfunction!(compare_values, module)?)?; + module.add_function(wrap_pyfunction!(sortable_key, module)?)?; + module.add_function(wrap_pyfunction!(max_bound, module)?)?; + module.add_function(wrap_pyfunction!(min_bound, module)?)?; + module.add_function(wrap_pyfunction!(floor_value, module)?)?; + module.add_function(wrap_pyfunction!(next_value, module)?)?; + module.add_function(wrap_pyfunction!(to_exclusive, module)?)?; + module.add_function(wrap_pyfunction!(to_inclusive, module)?)?; + module.add_function(wrap_pyfunction!(observed_end, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/values.rs new file mode 100644 index 0000000000..4c7e219b8b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/values.rs @@ -0,0 +1,91 @@ +//! Python `date`, `datetime` and `timedelta` objects read into and built from cursor values. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +pub(crate) const DATETIME_MODULE: &str = "datetime"; +const MICROS_PER_SECOND: i64 = 1_000_000; +const SECONDS_PER_DAY: i64 = 86_400; + +/// The year, month and day of a `date` or `datetime`. +pub(crate) fn civil_date(value: &Bound<'_, PyAny>) -> PyResult { + Ok(CivilDate { + year: value.getattr("year")?.extract()?, + month: value.getattr("month")?.extract()?, + day: value.getattr("day")?.extract()?, + }) +} + +/// The wall-clock fields of a `datetime`. +pub(crate) fn local_date_time(value: &Bound<'_, PyAny>) -> PyResult { + Ok(LocalDateTime { + date: civil_date(value)?, + hour: value.getattr("hour")?.extract()?, + minute: value.getattr("minute")?.extract()?, + second: value.getattr("second")?.extract()?, + microsecond: value.getattr("microsecond")?.extract()?, + }) +} + +/// A `timedelta` as whole microseconds. +pub(crate) fn timedelta_micros(delta: &Bound<'_, PyAny>) -> PyResult { + let days: i64 = delta.getattr("days")?.extract()?; + let seconds: i64 = delta.getattr("seconds")?.extract()?; + let microseconds: i64 = delta.getattr("microseconds")?.extract()?; + Ok((days * SECONDS_PER_DAY + seconds) * MICROS_PER_SECOND + microseconds) +} + +/// `template.replace(...)` with the wall time and fold of `local`, keeping type and `tzinfo`. +pub(crate) fn replaced<'py>( + template: &Bound<'py, PyAny>, + local: &LocalDateTime, + fold: bool, +) -> PyResult> { + let py: Python<'py> = template.py(); + call_with( + &template.getattr("replace")?, + &[ + ("year", local.date.year.into_pyobject(py)?.into_any()), + ("month", local.date.month.into_pyobject(py)?.into_any()), + ("day", local.date.day.into_pyobject(py)?.into_any()), + ("hour", local.hour.into_pyobject(py)?.into_any()), + ("minute", local.minute.into_pyobject(py)?.into_any()), + ("second", local.second.into_pyobject(py)?.into_any()), + ( + "microsecond", + local.microsecond.into_pyobject(py)?.into_any(), + ), + ("fold", u8::from(fold).into_pyobject(py)?.into_any()), + ], + ) +} + +/// A naive `datetime` holding `local`. +pub(crate) fn naive_date_time<'py>( + py: Python<'py>, + local: &LocalDateTime, + fold: bool, +) -> PyResult> { + let class: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + let value: Bound<'py, PyAny> = class.call1(( + local.date.year, + local.date.month, + local.date.day, + local.hour, + local.minute, + local.second, + local.microsecond, + ))?; + if !fold { + return Ok(value); + } + replaced(&value, local, fold) +} + +/// A `date` holding `date`. +pub(crate) fn plain_date<'py>(py: Python<'py>, date: &CivilDate) -> PyResult> { + imported(py, DATETIME_MODULE, "date")?.call1((date.year, date.month, date.day)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/mod.rs new file mode 100644 index 0000000000..3d59a8b199 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod registration; +pub(crate) mod run_debug; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/registration.rs new file mode 100644 index 0000000000..6ef424c33a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/registration.rs @@ -0,0 +1,9 @@ +//! Register the `sqb debug` command. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::debug_command::run_debug; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + run_debug::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs new file mode 100644 index 0000000000..84ad9cafc5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -0,0 +1,412 @@ +//! `sqb debug`: report the runtime, resolved configuration and connection health. + +use std::path::PathBuf; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::debug_report::main::debug_config_files::debug_config_files; +use sqlbuild_core::debug_report::main::debug_configuration_lines::debug_configuration_lines; +use sqlbuild_core::debug_report::main::debug_connection_lines::debug_connection_lines; +use sqlbuild_core::debug_report::main::debug_provider_lines::debug_provider_lines; +use sqlbuild_core::debug_report::main::debug_runtime_lines::debug_runtime_lines; +use sqlbuild_core::debug_report::main::format_debug_json::format_debug_json; +use sqlbuild_core::debug_report::main::format_debug_text::format_debug_text; +use sqlbuild_core::debug_report::models::{ + ConnectionCheck, ConnectionValue, DebugConfigFiles, DebugProjectFacts, DebugProvider, + DebugReport, DebugRuntime, WarehouseCheck, +}; + +use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::command_runtime::runtime_errors::{ + expected_resolution_error, resolution_message, +}; +use crate::bindings::_helpers::command_runtime::warehouses::{ + CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const DEBUG_OPTIONS: (&[&str], &[&str]) = ( + &[], + &[ + "project_dir", + "no_color", + "no_connection", + "selected_target", + "json_output", + ], +); + +/// The Python objects and options one debug invocation resolves against. +struct DebugContext<'py> { + project_dir: Bound<'py, PyAny>, + discovered_inputs: Bound<'py, PyAny>, + selected_target: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, +} + +fn runtime(py: Python<'_>) -> PyResult { + let platform: Bound<'_, PyModule> = py.import("platform")?; + let sqlbuild_version: Option = + match imported(py, "importlib.metadata", "version")?.call1(("sqlbuild",)) { + Ok(version) => Some(version.str()?.extract()?), + Err(error) + if error.is_instance( + py, + &imported(py, "importlib.metadata", "PackageNotFoundError")?, + ) => + { + None + } + Err(error) => return Err(error), + }; + Ok(DebugRuntime { + sqlbuild_version, + python_version: platform.call_method0("python_version")?.str()?.extract()?, + python_path: py.import("sys")?.getattr("executable")?.str()?.extract()?, + os_info: platform.call_method0("platform")?.str()?.extract()?, + }) +} + +/// A non-empty string value, as `_display_target_value` accepts it. +fn target_value(value: &Bound<'_, PyAny>) -> PyResult> { + if !value.is_instance_of::() { + return Ok(None); + } + let text: String = value.extract()?; + Ok((!text.is_empty()).then_some(text)) +} + +/// `str(path)` of a joined path, normalised as `pathlib` normalises it. +fn path_text(py: Python<'_>, path: &PathBuf) -> PyResult { + imported(py, "pathlib", "Path")? + .call1((path,))? + .str()? + .extract() +} + +fn optional_string(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +fn warehouse_checks( + context: &DebugContext<'_>, + (command_scope, base_connection): (&CommandScope<'_>, &Bound<'_, PyAny>), +) -> PyResult> { + let py: Python<'_> = context.adapter.py(); + let supports_session_warehouse: bool = context + .adapter + .getattr("supports_session_warehouse")? + .is_truthy()?; + let adapter_name: String = context.adapter.getattr("adapter_name")?.str()?.extract()?; + let inputs: WarehouseInputs<'_, '_> = WarehouseInputs { + discovered_inputs: &context.discovered_inputs, + selected_target: &context.selected_target, + }; + let mut checks: Vec = Vec::new(); + for group in imported(py, CONTRACT_TYPES_MODULE, "WarehouseGroup")?.try_iter()? { + let group: Bound<'_, PyAny> = group?; + let group_name: String = group.getattr("value")?.str()?.extract()?; + if !supports_session_warehouse { + if let Some(found) = find_authored(inputs, &group)? { + checks.push(WarehouseCheck::NotUsed { + group: group_name, + adapter_name: adapter_name.clone(), + source: found.authored.source(), + }); + } + continue; + } + let resolved: PyResult<(Option, String)> = resolve_warehouse( + Some(inputs), + (&group, command_scope.cli_warehouse.as_deref()), + (base_connection, &command_scope.cli_vars), + ); + checks.push(match resolved { + Ok((warehouse, source)) => WarehouseCheck::Resolved { + group: group_name, + warehouse: warehouse.filter(|warehouse| !warehouse.is_empty()), + source, + }, + Err(error) if expected_resolution_error(py, &error)? => WarehouseCheck::Unresolved { + group: group_name, + error: resolution_message(py, &error)?, + }, + Err(error) => return Err(error), + }); + } + Ok(checks) +} + +fn providers(discovered_inputs: &Bound<'_, PyAny>) -> PyResult> { + let mut providers: Vec = Vec::new(); + for provider in discovered_inputs.getattr("providers")?.try_iter()? { + let provider: Bound<'_, PyAny> = provider?; + providers.push(DebugProvider { + name: provider.getattr("name")?.str()?.extract()?, + relative_path: provider.getattr("relative_path")?.str()?.extract()?, + class_name: provider + .getattr("provider_class")? + .getattr("__name__")? + .str()? + .extract()?, + }); + } + Ok(providers) +} + +fn connection_settings(config: &Bound<'_, PyDict>) -> PyResult> { + let mut settings: Vec<(String, ConnectionValue)> = Vec::new(); + for (key, value) in config.iter() { + let shown: ConnectionValue = if value.is_none() { + ConnectionValue::Null + } else if value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + { + ConnectionValue::Scalar(value.str()?.extract()?) + } else { + ConnectionValue::Text(value.str()?.extract()?) + }; + settings.push((key.extract()?, shown)); + } + Ok(settings) +} + +/// The message of an ordinary exception; anything else, such as an interrupt, propagates. +fn check_failure(py: Python<'_>, error: PyErr) -> PyResult { + if !error.is_instance_of::(py) { + return Err(error); + } + error.value(py).str()?.extract() +} + +fn connection_check( + adapter: &Bound<'_, PyAny>, + connection_config: &Bound<'_, PyDict>, +) -> PyResult { + let py: Python<'_> = adapter.py(); + let connection: Bound<'_, PyAny> = match adapter.call_method1("connect", (connection_config,)) { + Ok(connection) => connection, + Err(error) => return Ok(ConnectionCheck::ConnectFailed(check_failure(py, error)?)), + }; + let queried: PyResult> = call_with( + &adapter.getattr("query")?, + &[ + ("connection", connection.clone()), + ("sql", PyString::new(py, "SELECT 1").into_any()), + ("limit", py.None().into_bound(py)), + ], + ); + let query_error: Option = match queried { + Ok(_) => None, + Err(error) if error.is_instance_of::(py) => { + adapter.call_method1("close", (&connection,))?; + Some(error.value(py).str()?.extract()?) + } + Err(error) => { + adapter.call_method1("close", (&connection,))?; + return Err(error); + } + }; + if query_error.is_none() { + adapter.call_method1("close", (&connection,))?; + } + Ok(ConnectionCheck::Connected { query_error }) +} + +/// The effective connection: the base connection with the command's warehouse applied. +fn effective_connection<'py>( + context: &DebugContext<'py>, + (adapter_name, base_connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = adapter_name.py(); + let applied: PyResult> = apply_warehouse( + base_connection, + (adapter_name, &context.project_dir), + Some(WarehouseInputs { + discovered_inputs: &context.discovered_inputs, + selected_target: &context.selected_target, + }), + ); + match applied { + Err(error) if expected_resolution_error(py, &error)? => Ok(base_connection.clone()), + applied => applied, + } +} + +fn build_report( + py: Python<'_>, + project_dir: &PathBuf, + (selected_target, check_connection): (Bound<'_, PyAny>, bool), +) -> PyResult { + let project_dir_object: Bound<'_, PyAny> = + imported(py, "pathlib", "Path")?.call1((project_dir,))?; + let discovered_inputs: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir_object.clone())], + )?; + let files: DebugConfigFiles = debug_config_files(project_dir); + let adapter_name: Bound<'_, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ( + "project_config", + discovered_inputs.getattr("project_config")?, + ), + ("local_config", discovered_inputs.getattr("local_config")?), + ], + )?; + let adapter: Bound<'_, PyAny> = call_with( + &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", project_dir_object.clone()), + ], + )?; + let context: DebugContext<'_> = DebugContext { + project_dir: project_dir_object, + discovered_inputs, + selected_target, + adapter, + }; + let command_scope: CommandScope<'_> = command_scope(py)?; + let unscoped: Bound<'_, PyAny> = call_with( + &imported(py, SCOPE_MODULE, "command_warehouse_scope")?, + &[ + ("command", py.None().into_bound(py)), + ("cli_warehouse", py.None().into_bound(py)), + ], + )?; + let base_connection: Bound<'_, PyAny> = within(&unscoped, |_| { + project_connection_config( + &context.discovered_inputs, + (&context.project_dir, &context.selected_target), + (&py.None().into_bound(py), true), + ) + })?; + let connection_config: Bound<'_, PyDict> = + effective_connection(&context, (&adapter_name, &base_connection))?.cast_into()?; + let namespace: Bound<'_, PyTuple> = call_with( + &imported(py, NATIVE_MODULE, "build_effective_target_namespace")?, + &[ + ("discovered_inputs", context.discovered_inputs.clone()), + ("selected_target", context.selected_target.clone()), + ( + "connection_database", + connection_config + .get_item("database")? + .unwrap_or_else(|| py.None().into_bound(py)), + ), + ( + "connection_schema", + connection_config + .get_item("schema")? + .unwrap_or_else(|| py.None().into_bound(py)), + ), + ( + "default_database", + context.adapter.call_method0("default_database")?, + ), + ( + "default_schema", + context.adapter.call_method0("default_schema")?, + ), + ], + )? + .cast_into()?; + let target_config: Bound<'_, PyAny> = namespace.get_item(1)?; + let connection_name: Option = if target_config.is_none() { + None + } else { + optional_string(&target_config.getattr("connection_name")?)? + }; + let facts: DebugProjectFacts = DebugProjectFacts { + project_file: path_text(py, &files.project_file)?, + local_file: files + .local_file + .as_ref() + .map(|path| path_text(py, path)) + .transpose()?, + project_name: context + .discovered_inputs + .getattr("project_config")? + .getattr("name")? + .str()? + .extract()?, + adapter_name: adapter_name.str()?.extract()?, + target_name: optional_string(&namespace.get_item(0)?)?.filter(|name| !name.is_empty()), + connection_name, + database: target_value(&namespace.get_item(2)?)?, + schema: target_value(&namespace.get_item(3)?)?, + }; + let warehouses: Vec = + warehouse_checks(&context, (&command_scope, &base_connection))?; + let check: ConnectionCheck = if check_connection { + connection_check(&context.adapter, &connection_config)? + } else { + ConnectionCheck::Skipped + }; + Ok(DebugReport { + runtime: debug_runtime_lines(&runtime(py)?), + configuration: debug_configuration_lines(&facts, &warehouses), + providers: debug_provider_lines(&providers(&context.discovered_inputs)?), + connection: debug_connection_lines(&connection_settings(&connection_config)?, &check), + }) +} + +/// `sqb debug`: validate project config and the active connection; `1` when a check failed. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_debug_command(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "run_debug_command", options, DEBUG_OPTIONS)?; + let project_dir: Bound<'_, PyAny> = arguments.value("project_dir")?; + let project_dir: PathBuf = if project_dir.is_none() { + imported(py, "pathlib", "Path")? + .call_method0("cwd")? + .extract()? + } else { + project_dir.extract()? + }; + let report: DebugReport = build_report( + py, + &project_dir, + ( + arguments.value("selected_target")?, + !arguments.flag("no_connection")?, + ), + )?; + let text: String = if arguments.flag("json_output")? { + format_debug_json(&report) + "\n" + } else { + let use_color: bool = !arguments.flag("no_color")? + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?; + format_debug_text(&report, CliStyle { use_color }) + }; + let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + stdout.call_method1("write", (text,))?; + stdout.call_method0("flush")?; + Ok(if report.success() { 0 } else { 1 }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_debug_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs new file mode 100644 index 0000000000..2bc4885d43 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -0,0 +1,386 @@ +//! `sqb freshness`: observe selected source freshness without writing state. + +use std::collections::BTreeSet; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::freshness_text::main::freshness_exit_code::freshness_exit_code; +use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; +use sqlbuild_core::freshness_text::main::render_freshness_text::render_freshness_text; +use sqlbuild_core::freshness_text::models::FreshnessSourceResult; + +use crate::bindings::_helpers::freshness::freshness_observe::{ + Observer, PreviousState, observe_freshness, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const CONNECTION_MODULE: &str = "sqlbuild._native"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; +const UPSTREAM_MODULE: &str = "sqlbuild.compiler.planner.main.selection._upstream"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; +const STATE_READ_MODULE: &str = "sqlbuild.compiler.source_freshness.main.read"; +const STATE_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +const STALE_REQUIRES_STATE_MESSAGE: &str = "freshness --fail-on-stale requires --state"; +const STALE_REQUIRES_STATE_CODE: &str = "C238"; + +fn imported<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { + py.import(module)?.getattr(name) +} + +fn keywords<'py>( + py: Python<'py>, + items: &[(&str, &Bound<'py, PyAny>)], +) -> PyResult> { + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in items { + dict.set_item(*name, value)?; + } + Ok(dict) +} + +fn user_error(py: Python<'_>, (message, code): (&str, &str)) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", code)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn write(py: Python<'_>, text: &str) -> PyResult<()> { + py.import("sys")? + .getattr("stdout")? + .call_method1("write", (text,))?; + Ok(()) +} + +/// Names of the sources among `keys` and their upstream closures. +fn source_names_for_keys( + graph: &Bound<'_, PyAny>, + keys: &Bound<'_, PyAny>, +) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + let py: Python<'_> = graph.py(); + let source_type: Bound<'_, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SOURCE")?; + let expand: Bound<'_, PyAny> = imported(py, UPSTREAM_MODULE, "expand_project_upstream_keys")?; + let upstream_deps: Bound<'_, PyAny> = graph.getattr("upstream_deps")?; + for key in keys.try_iter()? { + let key: Bound<'_, PyAny> = key?; + let upstream: Bound<'_, PyAny> = expand.call( + (), + Some(&keywords( + py, + &[("key", &key), ("upstream_deps", &upstream_deps)], + )?), + )?; + for candidate in std::iter::once(Ok(key.clone())).chain(upstream.try_iter()?) { + let candidate: Bound<'_, PyAny> = candidate?; + if candidate.getattr("resource_type")?.eq(&source_type)? { + names.insert(candidate.getattr("name")?.extract()?); + } + } + } + Ok(names) +} + +fn graph_keys<'py>( + graph: &Bound<'py, PyAny>, + selectors: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = graph.py(); + let empty: Bound<'py, PyAny> = PyTuple::empty(py).into_any(); + imported(py, GRAPH_SELECTION_MODULE, "select_project_graph")?.call( + (), + Some(&keywords( + py, + &[("graph", graph), ("select", selectors), ("exclude", &empty)], + )?), + ) +} + +/// Selected source names, sorted: selected or all keys and their upstream sources, minus excluded. +fn selected_source_names( + graph: &Bound<'_, PyAny>, + (select, exclude): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult> { + let selected_keys: Bound<'_, PyAny> = if select.is_truthy()? { + graph_keys(graph, select)? + } else { + graph.getattr("all_keys")?.call_method0("values")? + }; + let selected: BTreeSet = source_names_for_keys(graph, &selected_keys)?; + let excluded: BTreeSet = if exclude.is_truthy()? { + source_names_for_keys(graph, &graph_keys(graph, exclude)?)? + } else { + BTreeSet::new() + }; + Ok(selected.difference(&excluded).cloned().collect()) +} + +/// Latest direct freshness state records, by identity, across the project's target schemas. +fn previous_records<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let mut database: Option> = None; + let mut schemas: BTreeSet = BTreeSet::new(); + for collection in ["models", "seeds", "functions"] { + for item in project.getattr(collection)?.try_iter()? { + let destination: Bound<'py, PyAny> = item?.getattr("destination")?; + let destination_database: Bound<'py, PyAny> = destination.getattr("database")?; + if database.is_none() && !destination_database.is_none() { + database = Some(destination_database); + } + let schema: Bound<'py, PyAny> = destination.getattr("schema")?; + if !schema.is_none() { + schemas.insert(schema.extract()?); + } + } + } + let database: Bound<'py, PyAny> = database.unwrap_or_else(|| py.None().into_bound(py)); + let locations: Vec> = schemas + .iter() + .map(|schema| { + PyTuple::new( + py, + [ + database.clone(), + PyString::new(py, schema).into_any(), + PyString::new(py, STATE_TABLE_NAME).into_any(), + ], + ) + }) + .collect::>()?; + let lookup: Bound<'py, PyAny> = imported(py, RELATION_LOOKUP_MODULE, "build_relation_lookup")? + .call( + (), + Some(&keywords( + py, + &[ + ("adapter", adapter), + ("connection", connection), + ("locations", &PyTuple::new(py, locations)?.into_any()), + ], + )?), + )?; + let read: Bound<'py, PyAny> = imported(py, STATE_READ_MODULE, "read_latest_source_freshness")?; + let records: Bound<'py, PyDict> = PyDict::new(py); + for schema in &schemas { + let exists: Bound<'py, PyDict> = keywords(py, &[("database", &database)])?; + exists.set_item("schema", schema)?; + exists.set_item("name", STATE_TABLE_NAME)?; + let options: Bound<'py, PyDict> = keywords( + py, + &[ + ("connection", connection), + ("execute", &adapter.getattr("execute")?), + ( + "table_exists", + &lookup.call_method("exists", (), Some(&exists))?, + ), + ("database", &database), + ( + "render_qualified_name", + &adapter.getattr("render_qualified_name")?, + ), + ( + "render_read_latest_sql", + &adapter.getattr("render_read_latest_source_freshness_sql")?, + ), + ], + )?; + options.set_item("schema", schema)?; + records.update( + read.call((), Some(&options))? + .getattr("records")? + .downcast()?, + )?; + } + Ok(records) +} + +/// Connect, read previous state when requested, observe, and close the connection. +fn observe( + request: &Bound<'_, PyAny>, + (discovered, project_dir): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (adapter, graph): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + names: &[String], +) -> PyResult> { + let py: Python<'_> = adapter.py(); + let connection_config: Bound<'_, PyAny> = + imported(py, CONNECTION_MODULE, "resolve_project_connection_config")?.call( + (), + Some(&keywords( + py, + &[ + ("discovered_inputs", discovered), + ("project_dir", project_dir), + ("selected_target", &request.getattr("selected_target")?), + ("cli_vars", &request.getattr("cli_vars")?), + ], + )?), + )?; + let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (connection_config,))?; + let observed = (|| { + let project: Bound<'_, PyAny> = graph.getattr("project")?; + let by_identity: Option> = + if request.getattr("compare_state")?.extract::()? { + Some(previous_records(adapter, &connection, &project)?.into_any()) + } else { + None + }; + let sources: Vec> = project + .getattr("sources")? + .try_iter()? + .map(|source| source?.getattr("source_entry")) + .collect::>()?; + let observed_at: Bound<'_, PyAny> = imported(py, "datetime", "datetime")? + .call_method1("now", (imported(py, "datetime", "UTC")?,))?; + let observer: Observer<'_> = Observer { + adapter: adapter.clone(), + connection: connection.clone(), + observed_at, + }; + let state: PreviousState<'_> = PreviousState { by_identity }; + observe_freshness( + &observer, + (&PyTuple::new(py, sources)?.into_any(), names, &[]), + (project.getattr("run_id")?.extract()?, &state), + ) + })(); + let closed: PyResult> = adapter.call_method1("close", (&connection,)); + let results: Vec = observed?; + closed?; + Ok(results) +} + +fn write_json_output( + py: Python<'_>, + payload: &str, + (json_output, json_output_path): (bool, &Bound<'_, PyAny>), +) -> PyResult<()> { + let options: Bound<'_, PyDict> = keywords(py, &[("json_output_path", json_output_path)])?; + options.set_item("payload", format!("{payload}\n"))?; + options.set_item("json_output", json_output)?; + imported(py, NATIVE_MODULE, "write_execution_json_output")?.call((), Some(&options))?; + Ok(()) +} + +/// Run one `sqb freshness` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let fail_on_stale: bool = request.getattr("fail_on_stale")?.extract()?; + let fail_on_error: bool = request.getattr("fail_on_error")?.extract()?; + if fail_on_stale && !request.getattr("compare_state")?.extract::()? { + return Err(user_error( + py, + (STALE_REQUIRES_STATE_MESSAGE, STALE_REQUIRES_STATE_CODE), + )); + } + let project_dir: Bound<'_, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, + }; + let discovered: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "discover_project_inputs")? + .call((), Some(&keywords(py, &[("project_dir", &project_dir)])?))?; + let select: Bound<'_, PyAny> = request.getattr("select")?; + let exclude: Bound<'_, PyAny> = request.getattr("exclude")?; + imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?.call( + (), + Some(&keywords( + py, + &[ + ("select", &select), + ("exclude", &exclude), + ("discovered_inputs", &discovered), + ], + )?), + )?; + let adapter_name: Bound<'_, PyAny> = + imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?.call( + (), + Some(&keywords( + py, + &[ + ("project_config", &discovered.getattr("project_config")?), + ("local_config", &discovered.getattr("local_config")?), + ], + )?), + )?; + let adapter: Bound<'_, PyAny> = imported(py, ADAPTERS_MODULE, "resolve_adapter")?.call( + (), + Some(&keywords( + py, + &[ + ("adapter_name", &adapter_name), + ("project_dir", &project_dir), + ], + )?), + )?; + let graph: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "build_project_graph")?.call( + (), + Some(&keywords( + py, + &[ + ("discovered_inputs", &discovered), + ("adapter", &adapter), + ("selected_target", &request.getattr("selected_target")?), + ("no_sql_validation", &request.getattr("no_sql_validation")?), + ("cli_vars", &request.getattr("cli_vars")?), + ], + )?), + )?; + let names: Vec = selected_source_names(&graph, (&select, &exclude))?; + let results: Vec = if names.is_empty() { + Vec::new() + } else { + observe( + request, + (&discovered, &project_dir), + (&adapter, &graph), + &names, + )? + }; + let payload: String = + freshness_json(&results).map_err(|error| PyValueError::new_err(format!("{error:?}")))?; + let json_output_path: Bound<'_, PyAny> = request.getattr("json_output_path")?; + if request.getattr("json_output")?.extract::()? { + write_json_output(py, &payload, (true, &json_output_path))?; + } else { + if !json_output_path.is_none() { + write_json_output(py, &payload, (false, &json_output_path))?; + } + let use_color: bool = !request.getattr("no_color")?.extract::()? + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .extract::()?; + write(py, &render_freshness_text(&results, CliStyle { use_color }))?; + } + Ok(freshness_exit_code( + &results, + (fail_on_stale, fail_on_error), + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_freshness_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs new file mode 100644 index 0000000000..79536ecd32 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs @@ -0,0 +1,483 @@ +//! Observe the current freshness of selected sources and compare it with previous state. + +use std::collections::{HashMap, HashSet}; + +use pyo3::exceptions::{PyException, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use sqlbuild_core::freshness_text::main::freshness_status::freshness_status; +use sqlbuild_core::freshness_text::models::{ + FreshnessSourceResult, FreshnessStatus, StateComparison, +}; + +const FRESHNESS_PACKAGE: &str = "sqlbuild.compiler.source_freshness.main"; +const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; +const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; +const RUN_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.run_scope"; +const ADAPTER_STRATEGY: &str = "adapter"; +const UNAVAILABLE_REASON: &str = "unavailable"; +const OBSERVATION_RUN_ID: &str = "freshness"; +const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; +const MISSING_STATE_MESSAGE: &str = "previous source freshness state missing"; +const NOT_RETURNED_MESSAGE: &str = "freshness metadata was not returned"; + +/// The adapter, connection and observation time every source is observed with. +pub(crate) struct Observer<'py> { + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) observed_at: Bound<'py, PyAny>, +} + +/// Previous state records by identity; absent means no comparison. +pub(crate) struct PreviousState<'py> { + pub(crate) by_identity: Option>, +} + +fn freshness_function<'py>( + py: Python<'py>, + (module, name): (&str, &str), +) -> PyResult> { + py.import(format!("{FRESHNESS_PACKAGE}.{module}"))? + .getattr(name) +} + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +/// `str(exc)` of an `Exception`, or the error itself for anything else. +fn exception_message(py: Python<'_>, error: PyErr) -> PyResult { + if !error.is_instance_of::(py) { + return Err(error); + } + error.value(py).str()?.extract() +} + +fn target_result( + source: &Bound<'_, PyAny>, + (name, status): (&str, FreshnessStatus), + message: Option, +) -> PyResult { + let mut result: FreshnessSourceResult = FreshnessSourceResult::new(name, status); + result.target_database = optional_text(source.getattr("database")?)?; + result.target_schema = optional_text(source.getattr("schema")?)?; + result.target_name = optional_text(source.getattr("table")?)?; + result.message = message; + Ok(result) +} + +fn strategy(source: &Bound<'_, PyAny>) -> PyResult> { + let freshness: Bound<'_, PyAny> = source.getattr("freshness")?; + if freshness.is_none() { + return Ok(None); + } + optional_text(freshness.getattr("strategy")?) +} + +/// The source as observed: itself with freshness config, an adapter-metadata copy, or none. +fn observation_source<'py>( + adapter: &Bound<'py, PyAny>, + source: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = source.py(); + if !source.getattr("freshness")?.is_none() { + return Ok(Some(source.clone())); + } + if !source.getattr("expression")?.is_none() + || source.getattr("table")?.is_none() + || !adapter + .call_method0("supports_table_freshness_metadata")? + .is_truthy()? + { + return Ok(None); + } + let config: Bound<'py, PyDict> = PyDict::new(py); + config.set_item( + "strategy", + py.import(CONTRACT_TYPES)? + .getattr("SourceFreshnessStrategy")? + .getattr("ADAPTER")?, + )?; + let changes: Bound<'py, PyDict> = PyDict::new(py); + changes.set_item( + "freshness", + py.import(CONTRACT_MODELS)? + .getattr("SourceFreshnessConfig")? + .call((), Some(&config))?, + )?; + Ok(Some( + py.import("dataclasses")? + .getattr("replace")? + .call((source,), Some(&changes))?, + )) +} + +/// The state record of one observation, with its normalized version and hash. +fn record<'py>( + observation: &Bound<'py, PyAny>, + source: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = observation.py(); + let normalize: Bound<'py, PyDict> = PyDict::new(py); + normalize.set_item("value", observation.getattr("data_version")?)?; + normalize.set_item("value_kind", observation.getattr("value_kind")?)?; + let data_version: Bound<'py, PyAny> = freshness_function( + py, + ("normalization", "normalize_source_freshness_data_version"), + )? + .call((), Some(&normalize))?; + let hash: Bound<'py, PyDict> = PyDict::new(py); + hash.set_item("source_name", observation.getattr("source_name")?)?; + hash.set_item("strategy", observation.getattr("strategy")?)?; + hash.set_item("value_kind", observation.getattr("value_kind")?)?; + hash.set_item("data_version", &data_version)?; + let data_version_hash: Bound<'py, PyAny> = freshness_function( + py, + ("data_version_hash", "source_freshness_data_version_hash"), + )? + .call((), Some(&hash))?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("source_name", observation.getattr("source_name")?)?; + fields.set_item("target_database", source.getattr("database")?)?; + fields.set_item("target_schema", source.getattr("schema")?)?; + fields.set_item("target_name", source.getattr("table")?)?; + fields.set_item("run_id", OBSERVATION_RUN_ID)?; + fields.set_item( + "strategy", + observation.getattr("strategy")?.getattr("value")?, + )?; + fields.set_item( + "value_kind", + observation.getattr("value_kind")?.getattr("value")?, + )?; + fields.set_item("data_version", &data_version)?; + fields.set_item("data_version_hash", data_version_hash)?; + fields.set_item("observed_at", observation.getattr("observed_at")?)?; + py.import(FRESHNESS_MODELS)? + .getattr("SourceFreshnessRecord")? + .call((), Some(&fields)) +} + +fn previous_record<'py>( + state: &PreviousState<'py>, + record: &Bound<'py, PyAny>, +) -> PyResult>> { + let Some(lookup) = &state.by_identity else { + return Ok(None); + }; + let found: Bound<'py, PyAny> = lookup.call_method1("get", (record.getattr("identity")?,))?; + Ok(found.is_truthy()?.then_some(found)) +} + +/// The command result of one successful observation. +fn observed_result( + (name, source, observation): (&str, &Bound<'_, PyAny>, &Bound<'_, PyAny>), + state: &PreviousState<'_>, +) -> PyResult { + let py: Python<'_> = source.py(); + let current: Bound<'_, PyAny> = record(observation, source)?; + let freshness: Bound<'_, PyAny> = source.getattr("freshness")?; + let (lag_tolerance, policy) = if freshness.is_none() { + (py.None().into_bound(py), py.None().into_bound(py)) + } else { + ( + freshness.getattr("lag_tolerance")?, + freshness.getattr("age_policy")?, + ) + }; + let age: Bound<'_, PyDict> = PyDict::new(py); + age.set_item("policy", policy)?; + age.set_item("data_version", observation.getattr("data_version")?)?; + age.set_item("observed_at", observation.getattr("observed_at")?)?; + let age_status: Bound<'_, PyAny> = + freshness_function(py, ("age_policy", "evaluate_source_freshness_age_policy"))? + .call((), Some(&age))?; + let previous: Option> = previous_record(state, ¤t)?; + let comparison: StateComparison = match &previous { + None if state.by_identity.is_some() => StateComparison::Missing, + None => StateComparison::NotCompared, + Some(previous) => { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("previous_record", previous)?; + options.set_item("current_record", ¤t)?; + options.set_item("lag_tolerance", &lag_tolerance)?; + let equivalent: bool = freshness_function( + py, + ("record_equivalence", "source_freshness_records_equivalent"), + )? + .call((), Some(&options))? + .is_truthy()?; + if equivalent { + StateComparison::Equivalent { + same_hash: previous + .getattr("data_version_hash")? + .eq(current.getattr("data_version_hash")?)?, + } + } else { + StateComparison::Different + } + } + }; + let mut result: FreshnessSourceResult = + FreshnessSourceResult::new(name, freshness_status(comparison)); + result.strategy = optional_text(current.getattr("strategy")?)?; + result.value_kind = optional_text(current.getattr("value_kind")?)?; + result.current_data_version = optional_text(current.getattr("data_version")?)?; + result.previous_data_version = match &previous { + Some(previous) => optional_text(previous.getattr("data_version")?)?, + None => None, + }; + result.lag_tolerance = optional_text(lag_tolerance)?; + result.target_database = optional_text(current.getattr("target_database")?)?; + result.target_schema = optional_text(current.getattr("target_schema")?)?; + result.target_name = optional_text(current.getattr("target_name")?)?; + if comparison == StateComparison::Missing { + result.message = Some(MISSING_STATE_MESSAGE.to_owned()); + } + result.age_status = if age_status.is_none() { + None + } else { + Some(age_status.getattr("value")?.extract()?) + }; + Ok(result) +} + +/// Adapter-strategy observations in one batch, or the message of the batch failure. +fn adapter_batch<'py>( + observer: &Observer<'py>, + sources: &[Bound<'py, PyAny>], +) -> PyResult, String>> { + let py: Python<'py> = observer.adapter.py(); + let batch_class: Bound<'py, PyAny> = py + .import(FRESHNESS_MODELS)? + .getattr("AdapterSourceFreshnessBatch")?; + if sources.is_empty() { + return Ok(Ok(batch_class.call0()?)); + } + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("adapter", &observer.adapter)?; + options.set_item("connection", &observer.connection)?; + options.set_item("sources", PyTuple::new(py, sources)?)?; + options.set_item("observed_at", &observer.observed_at)?; + match freshness_function( + py, + ("adapter_observation", "observe_adapter_sources_freshness"), + )? + .call((), Some(&options)) + { + Ok(batch) => Ok(Ok(batch)), + Err(error) => Ok(Err(exception_message(py, error)?)), + } +} + +/// One adapter-strategy source's observation, or its unknown or error result. +fn adapter_observation<'py>( + batch: &Result, String>, + (name, source): (&str, &Bound<'py, PyAny>), +) -> PyResult, FreshnessSourceResult>> { + let batch: &Bound<'py, PyAny> = match batch { + Ok(batch) => batch, + Err(message) => { + return Ok(Err(target_result( + source, + (name, FreshnessStatus::Error), + Some(message.clone()), + )?)); + } + }; + let unknown: Bound<'py, PyAny> = batch.getattr("unknown")?.call_method1("get", (name,))?; + if !unknown.is_none() { + let reason: String = unknown.getattr("reason")?.str()?.extract()?; + let status: FreshnessStatus = if reason == UNAVAILABLE_REASON { + FreshnessStatus::Unknown + } else { + FreshnessStatus::Error + }; + return Ok(Err(target_result( + source, + (name, status), + optional_text(unknown.getattr("message")?)?, + )?)); + } + let observation: Bound<'py, PyAny> = batch + .getattr("observations")? + .call_method1("get", (name,))?; + if observation.is_none() { + return Ok(Err(target_result( + source, + (name, FreshnessStatus::Error), + Some(NOT_RETURNED_MESSAGE.to_owned()), + )?)); + } + Ok(Ok(observation)) +} + +/// One configured-strategy source's observation, or its error result. +fn configured_observation<'py>( + observer: &Observer<'py>, + (name, source): (&str, &Bound<'py, PyAny>), +) -> PyResult, FreshnessSourceResult>> { + let py: Python<'py> = source.py(); + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("adapter", &observer.adapter)?; + options.set_item("connection", &observer.connection)?; + options.set_item("source", source)?; + options.set_item("observed_at", &observer.observed_at)?; + match freshness_function(py, ("observation", "observe_configured_source_freshness"))? + .call((), Some(&options)) + { + Ok(observation) => Ok(Ok(observation)), + Err(error) => Ok(Err(target_result( + source, + (name, FreshnessStatus::Error), + Some(exception_message(py, error)?), + )?)), + } +} + +fn observe_sources( + observer: &Observer<'_>, + sources: &[Bound<'_, PyAny>], + state: &PreviousState<'_>, +) -> PyResult> { + let mut results: Vec = Vec::new(); + let mut observed: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); + let mut positions: HashMap = HashMap::new(); + let mut adapter_sources: Vec> = Vec::new(); + for source in sources { + let name: String = source.getattr("name")?.extract()?; + let Some(observed_source) = observation_source(&observer.adapter, source)? else { + results.push(target_result( + source, + (&name, FreshnessStatus::Unknown), + Some(NO_CONFIG_MESSAGE.to_owned()), + )?); + continue; + }; + if strategy(&observed_source)?.as_deref() == Some(ADAPTER_STRATEGY) { + adapter_sources.push(observed_source.clone()); + } + match positions.get(&name) { + Some(&position) => observed[position].1 = observed_source, + None => { + positions.insert(name.clone(), observed.len()); + observed.push((name, observed_source)); + } + } + } + let batch = adapter_batch(observer, &adapter_sources)?; + for (name, source) in &observed { + let observation = if strategy(source)?.as_deref() == Some(ADAPTER_STRATEGY) { + adapter_observation(&batch, (name, source))? + } else { + configured_observation(observer, (name, source))? + }; + match observation { + Ok(observation) => results.push(observed_result((name, source, &observation), state)?), + Err(result) => results.push(result), + } + } + results.sort_by(|left, right| left.name.cmp(&right.name)); + Ok(results) +} + +/// Observe the selected sources inside the run's observability scope. +pub(crate) fn observe_freshness( + observer: &Observer<'_>, + (sources, select, exclude): (&Bound<'_, PyAny>, &[String], &[String]), + (run_id, state): (Option, &PreviousState<'_>), +) -> PyResult> { + let py: Python<'_> = observer.adapter.py(); + let selected: HashSet<&str> = select.iter().map(String::as_str).collect(); + let excluded: HashSet<&str> = exclude.iter().map(String::as_str).collect(); + let mut chosen: Vec> = Vec::new(); + for source in sources.try_iter()? { + let source: Bound<'_, PyAny> = source?; + let name: String = source.getattr("name")?.extract()?; + if (selected.is_empty() || selected.contains(name.as_str())) + && !excluded.contains(name.as_str()) + { + chosen.push(source); + } + } + let Some(run_id) = run_id else { + return observe_sources(observer, &chosen, state); + }; + let scope: Bound<'_, PyAny> = py + .import(RUN_SCOPE_MODULE)? + .getattr("run_scope")? + .call1((run_id,))?; + scope.call_method0("__enter__")?; + match observe_sources(observer, &chosen, state) { + Ok(results) => { + scope.call_method1("__exit__", (py.None(), py.None(), py.None()))?; + Ok(results) + } + Err(error) => { + scope.call_method1( + "__exit__", + (error.get_type(py), error.value(py), error.traceback(py)), + )?; + Err(error) + } + } +} + +/// Observe source freshness and return the `sqb freshness --json` payload. +#[pyfunction] +#[pyo3(signature = (**options))] +fn observe_source_freshness_json( + py: Python<'_>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "observe_source_freshness_json", + options, + ( + &["adapter", "connection", "sources", "observed_at"], + &["select", "exclude", "run_id", "previous_records"], + ), + )?; + let names = |name: &str| -> PyResult> { + let value: Bound<'_, PyAny> = arguments.value(name)?; + if value.is_none() { + return Ok(Vec::new()); + } + value.extract() + }; + let observer: Observer<'_> = Observer { + adapter: arguments.value("adapter")?, + connection: arguments.value("connection")?, + observed_at: arguments.value("observed_at")?, + }; + let previous: Bound<'_, PyAny> = arguments.value("previous_records")?; + let state: PreviousState<'_> = PreviousState { + by_identity: (!previous.is_none()).then_some(previous), + }; + let results: Vec = observe_freshness( + &observer, + ( + &arguments.value("sources")?, + &names("select")?, + &names("exclude")?, + ), + (arguments.value("run_id")?.extract()?, &state), + )?; + freshness_json(&results).map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(observe_source_freshness_json, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs new file mode 100644 index 0000000000..a2e1f90b98 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod freshness_command; +pub(crate) mod freshness_observe; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs new file mode 100644 index 0000000000..0f2cffdbe0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs @@ -0,0 +1,10 @@ +//! Register the native `sqb freshness` command and its observation entry. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::freshness::{freshness_command, freshness_observe}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + freshness_command::register(module)?; + freshness_observe::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs new file mode 100644 index 0000000000..6fe5dbe9fc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -0,0 +1,315 @@ +//! `sqb janitor`: preview stale warehouse objects, confirm, and clean them up. + +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::janitor_text::errors::JanitorSettingsError; +use sqlbuild_core::janitor_text::main::janitor_completion_text::janitor_completion_text; +use sqlbuild_core::janitor_text::main::janitor_confirmation_phrase::janitor_confirmation_phrase; +use sqlbuild_core::janitor_text::main::janitor_confirmation_prompt::janitor_confirmation_prompt; +use sqlbuild_core::janitor_text::main::janitor_disabled_text::janitor_disabled_text; +use sqlbuild_core::janitor_text::main::janitor_plan_has_work::janitor_plan_has_work; +use sqlbuild_core::janitor_text::main::janitor_preview_complete_text::janitor_preview_complete_text; +use sqlbuild_core::janitor_text::main::janitor_preview_start_text::janitor_preview_start_text; +use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; +use sqlbuild_core::janitor_text::main::validate_janitor_settings::validate_janitor_settings; +use sqlbuild_core::janitor_text::models::JanitorPlanText; + +use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; +const RUNTIME_PACKAGE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime"; +const COMMAND_NAME: &str = "janitor"; + +fn imported<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { + py.import(module)?.getattr(name) +} + +/// One of the janitor runtime phases, which reach the adapter and executor. +fn runtime<'py>(py: Python<'py>, (module, name): (&str, &str)) -> PyResult> { + imported(py, &format!("{RUNTIME_PACKAGE}.{module}"), name) +} + +fn keywords<'py>( + py: Python<'py>, + items: &[(&str, &Bound<'py, PyAny>)], +) -> PyResult> { + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in items { + dict.set_item(*name, value)?; + } + Ok(dict) +} + +fn settings_error(py: Python<'_>, error: JanitorSettingsError) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", error.code)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((error.message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn write(py: Python<'_>, text: &str) -> PyResult<()> { + py.import("sys")? + .getattr("stdout")? + .call_method1("write", (text,))?; + Ok(()) +} + +fn flush(py: Python<'_>) -> PyResult<()> { + py.import("sys")?.getattr("stdout")?.call_method0("flush")?; + Ok(()) +} + +/// Discover the project and resolve the target and output context. +fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult> { + let project_dir: Bound<'py, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, + }; + let discovered: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "discover_project_inputs")? + .call((), Some(&keywords(py, &[("project_dir", &project_dir)])?))?; + let as_target: Bound<'py, PyAny> = request.getattr("as_target")?; + let preview: Bound<'py, PyDict> = keywords( + py, + &[ + ("discovered_inputs", &discovered), + ("as_target", &as_target), + ], + )?; + preview.set_item("command_name", COMMAND_NAME)?; + imported(py, PREVIEW_TARGET_MODULE, "validate_preview_target")?.call((), Some(&preview))?; + let selected_target: Bound<'py, PyAny> = request.getattr("selected_target")?; + let active_target: Bound<'py, PyAny> = imported(py, TARGET_NAME_MODULE, "resolve_target_name")? + .call( + (), + Some(&keywords( + py, + &[ + ("project_config", &discovered.getattr("project_config")?), + ("local_config", &discovered.getattr("local_config")?), + ("selected_target", &selected_target), + ], + )?), + )?; + let use_color: bool = !request.getattr("no_color")?.extract::()? + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .extract::()?; + let options: Bound<'py, PyDict> = keywords( + py, + &[ + ("effective_project_dir", &project_dir), + ("discovered_inputs", &discovered), + ("selected_target", &selected_target), + ("active_target_name", &active_target), + ("as_target", &as_target), + ], + )?; + options.set_item("use_color", use_color)?; + imported(py, COMMAND_MODELS, "JanitorInvocation")?.call((), Some(&options)) +} + +/// Effective retention settings, validated. +fn settings<'py>( + py: Python<'py>, + request: &Bound<'py, PyAny>, + janitor: &Bound<'py, PyAny>, +) -> PyResult> { + let retention_days: Bound<'py, PyAny> = + imported(py, RETENTION_MODULE, "resolve_janitor_retention_days")?.call( + (), + Some(&keywords( + py, + &[ + ("override", &request.getattr("retention_days")?), + ("configured", &janitor.getattr("retention_days")?), + ], + )?), + )?; + validate_janitor_settings(retention_days.extract()?, 0) + .map_err(|error| settings_error(py, error))?; + let requested: Bound<'py, PyAny> = request.getattr("direct_state_history_versions")?; + let history: Bound<'py, PyAny> = if requested.is_none() { + janitor.getattr("direct_state_history_versions")? + } else { + requested + }; + validate_janitor_settings(0, history.extract()?).map_err(|error| settings_error(py, error))?; + let options: Bound<'py, PyDict> = keywords( + py, + &[ + ("retention_days", &retention_days), + ("direct_state_history_versions", &history), + ( + "archive_retention_days", + &janitor.getattr("archive_retention_days")?, + ), + ( + "drop_old_name_views", + &request.getattr("drop_old_name_views")?, + ), + ], + )?; + imported(py, COMMAND_MODELS, "JanitorSettings")?.call((), Some(&options)) +} + +/// Ask for the exact confirmation phrase; an interrupt cancels. +fn confirmed(py: Python<'_>, plan: &JanitorPlanText) -> PyResult { + write(py, &janitor_confirmation_prompt(plan))?; + flush(py)?; + match imported(py, "builtins", "input")?.call0() { + Ok(response) => Ok(response.extract::()? == janitor_confirmation_phrase(plan)), + Err(error) if error.is_instance_of::(py) => { + write(py, "\n")?; + Ok(false) + } + Err(error) => Err(error), + } +} + +/// Plan, preview, confirm and clean up with an open warehouse connection. +fn clean( + py: Python<'_>, + request: &Bound<'_, PyAny>, + (invocation, settings): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (compile_context, connection_context): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let style: CliStyle = CliStyle { + use_color: invocation.getattr("use_color")?.extract()?, + }; + let planning: Bound<'_, PyAny> = runtime(py, ("planning", "build_janitor_execution_plan"))? + .call( + (), + Some(&keywords( + py, + &[ + ("invocation", invocation), + ("settings", settings), + ("compile_context", compile_context), + ("connection_context", connection_context), + ], + )?), + )?; + let plan: JanitorPlanText = janitor_plan_text(&planning.getattr("plan")?)?; + write(py, &render_janitor_plan(&plan, style))?; + let blocked: i32 = i32::from(!plan.blocked_schemas.is_empty()); + let as_target: Option = invocation.getattr("as_target")?.extract()?; + if let Some(as_target) = as_target { + let active: Option = invocation.getattr("active_target_name")?.extract()?; + write( + py, + &janitor_preview_complete_text(&as_target, active.as_deref(), style), + )?; + return Ok(blocked); + } + if blocked != 0 { + return Ok(1); + } + if !janitor_plan_has_work(&plan) { + return Ok(0); + } + if !request.getattr("auto_approve")?.extract::()? && !confirmed(py, &plan)? { + write(py, "Janitor cancelled.\n")?; + return Ok(1); + } + let result: Bound<'_, PyAny> = runtime(py, ("execution", "execute_janitor_cleanup"))?.call( + (), + Some(&keywords( + py, + &[ + ("compile_context", compile_context), + ("connection_context", connection_context), + ("planning_result", &planning), + ], + )?), + )?; + write( + py, + &janitor_completion_text(&janitor_result_counts(&result)?, style), + )?; + Ok(0) +} + +/// Run one `sqb janitor` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let invocation: Bound<'_, PyAny> = invocation(py, request)?; + let style: CliStyle = CliStyle { + use_color: invocation.getattr("use_color")?.extract()?, + }; + let janitor: Bound<'_, PyAny> = invocation + .getattr("discovered_inputs")? + .getattr("project_config")? + .getattr("janitor")?; + if !janitor.getattr("enabled")?.is_truthy()? { + write(py, &janitor_disabled_text(style))?; + return Ok(0); + } + let settings: Bound<'_, PyAny> = settings(py, request, &janitor)?; + let as_target: Option = invocation.getattr("as_target")?.extract()?; + if let Some(as_target) = &as_target { + let active: Option = invocation.getattr("active_target_name")?.extract()?; + write( + py, + &janitor_preview_start_text(as_target, active.as_deref()), + )?; + flush(py)?; + } + let compile_context: Bound<'_, PyAny> = + runtime(py, ("compilation", "compile_janitor_project"))? + .call((), Some(&keywords(py, &[("invocation", &invocation)])?))?; + let connection_context: Bound<'_, PyAny> = + runtime(py, ("connections", "connect_janitor_warehouse"))?.call( + (), + Some(&keywords( + py, + &[ + ("invocation", &invocation), + ("compile_context", &compile_context), + ], + )?), + )?; + let outcome: PyResult = clean( + py, + request, + (&invocation, &settings), + (&compile_context, &connection_context), + ); + let closed: PyResult> = + runtime(py, ("connections", "close_janitor_warehouse")).and_then(|close| { + close.call( + (), + Some(&keywords( + py, + &[ + ("compile_context", &compile_context), + ("connection_context", &connection_context), + ], + )?), + ) + }); + let code: i32 = outcome?; + closed?; + Ok(code) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_janitor_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs new file mode 100644 index 0000000000..7b3b8d4f3b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs @@ -0,0 +1,215 @@ +//! Display facts of the executor's janitor plan and execution result. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::janitor_text::main::janitor_completion_text::janitor_completion_text; +use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; +use sqlbuild_core::janitor_text::models::{ + ArchiveCandidateText, ArchivedRelationText, JanitorPlanText, JanitorResultCounts, + OldNameViewText, SourceSchemaText, +}; + +const TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S"; +const DATE_FORMAT: &str = "%Y-%m-%d"; + +fn display_name(item: &Bound<'_, PyAny>) -> PyResult { + item.call_method0("display_name")?.extract() +} + +fn key_name(item: &Bound<'_, PyAny>) -> PyResult { + display_name(&item.getattr("key")?) +} + +/// `format(value, pattern)`, as an f-string format spec renders a datetime. +fn formatted(value: &Bound<'_, PyAny>, pattern: &str) -> PyResult { + value + .py() + .import("builtins")? + .getattr("format")? + .call1((value, pattern))? + .extract() +} + +/// Whole days between `value` and the planning time, or none when either is unknown. +fn age_days(value: &Bound<'_, PyAny>, planned_at: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() || planned_at.is_none() { + return Ok(None); + } + Ok(Some(planned_at.sub(value)?.getattr("days")?.extract()?)) +} + +fn names(items: &Bound<'_, PyAny>) -> PyResult> { + items.try_iter()?.map(|item| key_name(&item?)).collect() +} + +fn archived( + item: &Bound<'_, PyAny>, + planned_at: &Bound<'_, PyAny>, +) -> PyResult { + let archived_at: Bound<'_, PyAny> = item.getattr("archived_at")?; + Ok(ArchivedRelationText { + name: key_name(item)?, + archived_in_run: !item.getattr("original_key")?.is_none(), + archived_at: formatted(&archived_at, TIMESTAMP_FORMAT)?, + age_days: age_days(&archived_at, planned_at)?, + expires_at: formatted(&item.getattr("expires_at")?, TIMESTAMP_FORMAT)?, + }) +} + +fn archived_list( + items: &Bound<'_, PyAny>, + planned_at: &Bound<'_, PyAny>, +) -> PyResult> { + items + .try_iter()? + .map(|item| archived(&item?, planned_at)) + .collect() +} + +fn old_name_views(views: &Bound<'_, PyAny>) -> PyResult> { + let mut read: Vec = Vec::new(); + for view in views.try_iter()? { + let view: Bound<'_, PyAny> = view?; + let expires_at: Bound<'_, PyAny> = view.getattr("expires_at")?; + let drop_reason: Bound<'_, PyAny> = view.getattr("drop_reason")?; + read.push(OldNameViewText { + name: key_name(&view)?, + destination_model: view.getattr("destination_model")?.str()?.extract()?, + expires_on: if expires_at.is_none() { + None + } else { + Some(formatted(&expires_at, DATE_FORMAT)?) + }, + drop_reason: if drop_reason.is_none() { + None + } else { + Some(drop_reason.str()?.extract()?) + }, + occupied: view.getattr("occupied")?.is_truthy()?, + claimed_by: view.getattr("claimed_by")?.extract()?, + }); + } + Ok(read) +} + +fn source_schemas(schemas: &Bound<'_, PyAny>, blocked: bool) -> PyResult> { + let mut read: Vec = Vec::new(); + for schema in schemas.try_iter()? { + let schema: Bound<'_, PyAny> = schema?; + let (suppressed_candidates, suppressed_archive_deletions) = if blocked { + ( + names(&schema.getattr("suppressed_candidates")?)?, + names(&schema.getattr("suppressed_archive_deletions")?)?, + ) + } else { + (Vec::new(), Vec::new()) + }; + read.push(SourceSchemaText { + name: display_name(&schema)?, + source_names: schema + .getattr("source_names")? + .try_iter()? + .map(|name| name?.str()?.extract()) + .collect::>()?, + suppressed_candidates, + suppressed_archive_deletions, + }); + } + Ok(read) +} + +/// The display facts of one executor `JanitorPlan`. +pub(crate) fn janitor_plan_text(plan: &Bound<'_, PyAny>) -> PyResult { + let planned_at: Bound<'_, PyAny> = plan.getattr("planned_at")?; + let mut archive_candidates: Vec = Vec::new(); + for candidate in plan.getattr("archive_candidates")?.try_iter()? { + let candidate: Bound<'_, PyAny> = candidate?; + let expires_at: Bound<'_, PyAny> = candidate.getattr("expires_at")?; + archive_candidates.push(ArchiveCandidateText { + name: key_name(&candidate)?, + archive_name: display_name(&candidate.getattr("archive_key")?)?, + age_days: age_days(&candidate.getattr("age_timestamp")?, &planned_at)?, + expires_now: expires_at.le(candidate.getattr("archived_at")?)?, + expires_at: formatted(&expires_at, TIMESTAMP_FORMAT)?, + }); + } + let mut state_prunes: Vec<(String, i64)> = Vec::new(); + for candidate in plan.getattr("direct_state_prune_candidates")?.try_iter()? { + let candidate: Bound<'_, PyAny> = candidate?; + state_prunes.push(( + display_name(&candidate)?, + candidate.getattr("retain_versions")?.extract()?, + )); + } + let mut skipped_relations: Vec<(String, String)> = Vec::new(); + for skipped in plan.getattr("skipped_relations")?.try_iter()? { + let skipped: Bound<'_, PyAny> = skipped?; + skipped_relations.push(( + key_name(&skipped)?, + skipped.getattr("reason")?.str()?.extract()?, + )); + } + let views: Bound<'_, PyAny> = plan.getattr("old_name_views")?; + Ok(JanitorPlanText { + target_name: plan.getattr("target_name")?.extract()?, + retention_days: plan.getattr("retention_days")?.extract()?, + direct_mode: plan.getattr("direct_mode")?.is_truthy()?, + archive_retention_days: plan.getattr("archive_retention_days")?.extract()?, + age_metadata_supported: plan.getattr("age_metadata_supported")?.is_truthy()?, + scanned_schema_count: plan.getattr("scanned_schema_count")?.extract()?, + candidates: names(&plan.getattr("candidates")?)?, + archive_candidates, + archive_deletions: archived_list( + &plan.getattr("archive_deletion_candidates")?, + &planned_at, + )?, + retained_archives: archived_list(&plan.getattr("retained_archives")?, &planned_at)?, + old_name_drops: old_name_views(&views.getattr("drops")?)?, + old_name_live: old_name_views(&views.getattr("live")?)?, + old_name_missing: old_name_views(&views.getattr("missing")?)?, + query_artifacts: names(&plan.getattr("query_diff_artifact_candidates")?)?, + state_prunes, + skipped_relations, + skipped_schemas: source_schemas(&plan.getattr("skipped_schemas")?, false)?, + blocked_schemas: source_schemas(&plan.getattr("blocked_schemas")?, true)?, + }) +} + +/// How many items an executor `JanitorExecutionResult` changed. +pub(crate) fn janitor_result_counts(result: &Bound<'_, PyAny>) -> PyResult { + let count = |name: &str| -> PyResult { result.getattr(name)?.len() }; + Ok(JanitorResultCounts { + archived: count("archived")?, + deleted: count("deleted")?, + deleted_archives: count("deleted_archives")?, + deleted_query_artifacts: count("deleted_query_diff_artifacts")?, + dropped_old_name_views: count("dropped_old_name_views")?, + pruned_direct_state: count("pruned_direct_state")?, + }) +} + +/// Render one executor janitor plan as the `sqb janitor` preview. +#[pyfunction] +#[pyo3(name = "render_janitor_plan", signature = (*, plan, use_color = false))] +fn render_janitor_plan_text(plan: &Bound<'_, PyAny>, use_color: bool) -> PyResult { + Ok(render_janitor_plan( + &janitor_plan_text(plan)?, + CliStyle { use_color }, + )) +} + +/// Render one executor janitor result as the `sqb janitor` completion line. +#[pyfunction] +#[pyo3(name = "render_janitor_completion", signature = (*, result, use_color = false))] +fn render_janitor_completion_text(result: &Bound<'_, PyAny>, use_color: bool) -> PyResult { + Ok(janitor_completion_text( + &janitor_result_counts(result)?, + CliStyle { use_color }, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_janitor_plan_text, module)?)?; + module.add_function(wrap_pyfunction!(render_janitor_completion_text, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs new file mode 100644 index 0000000000..af44f2ecdc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod janitor_command; +pub(crate) mod janitor_plan; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs new file mode 100644 index 0000000000..4bbcf0e7e1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs @@ -0,0 +1,10 @@ +//! Register the native `sqb janitor` command and its plan rendering. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::janitor::{janitor_command, janitor_plan}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + janitor_command::register(module)?; + janitor_plan::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs new file mode 100644 index 0000000000..24da1a8fc2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs @@ -0,0 +1,468 @@ +//! The disposable SQLite cache of relation lineage, read and written through Python's `sqlite3`. + +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyResult, Python}; +use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyList, PyTuple}; +use sqlbuild_analysis::lineage_command::models::{LineageIndex, LineageKey, LineageNode}; + +const CACHE_SCHEMA_VERSION: &str = "1"; +const CACHE_ALGORITHM_VERSION: &str = "relation-lineage-graph-v4"; +const CACHE_RELATIVE_PATH: [&str; 5] = [ + "target", + "cache", + "lineage-native-v1", + "v1", + "structural-graph.sqlite3", +]; +const CACHE_MAX_BYTES: u64 = 50_000_000; +const CACHE_MAX_ROWS: usize = 1_000_000; +const SQLITE_TIMEOUT_SECONDS: f64 = 5.0; +/// The first Python whose `rglob` skips a directory listing that fails mid-way. +const FIRST_PYTHON_SKIPPING_INTERRUPTED_LISTINGS: (u8, u8) = (3, 13); +const UNKNOWN_VERSION: &str = "0+unknown"; +/// Exceptions that make the cache a miss, or skip publishing it. +const CACHE_FAULTS: [(&str, &str); 4] = [ + ("builtins", "KeyError"), + ("builtins", "OSError"), + ("builtins", "TypeError"), + ("builtins", "ValueError"), +]; +const FINGERPRINT_FAULTS: [&str; 3] = ["TypeError", "UnicodeError", "ValueError"]; +const SCHEMA_SQL: &str = " + CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE resources ( + resource_type TEXT NOT NULL, + resource_name TEXT NOT NULL, + relative_path TEXT, + qualified_name TEXT, + PRIMARY KEY (resource_type, resource_name) + ); + CREATE TABLE lookups ( + lookup_name TEXT PRIMARY KEY, + resource_type TEXT NOT NULL, + resource_name TEXT NOT NULL + ); + CREATE TABLE edges ( + upstream_type TEXT NOT NULL, + upstream_name TEXT NOT NULL, + downstream_type TEXT NOT NULL, + downstream_name TEXT NOT NULL, + PRIMARY KEY (upstream_type, upstream_name, downstream_type, downstream_name) + ); + CREATE INDEX edges_downstream ON edges (downstream_type, downstream_name); + CREATE INDEX edges_upstream ON edges (upstream_type, upstream_name); + CREATE TABLE tags ( + tag TEXT NOT NULL, + resource_type TEXT NOT NULL, + resource_name TEXT NOT NULL, + PRIMARY KEY (tag, resource_type, resource_name) + ); + CREATE TABLE paths ( + resource_type TEXT NOT NULL, + resource_name TEXT NOT NULL, + folder TEXT NOT NULL, + PRIMARY KEY (resource_type, resource_name) + ); + "; + +type ResourceRow = (String, String, Option, Option); +type EdgeRow = (String, String, String, String); +type TripleRow = (String, String, String); + +fn cache_path(project_dir: &Path) -> PathBuf { + CACHE_RELATIVE_PATH + .iter() + .fold(project_dir.to_path_buf(), |path, part| path.join(part)) +} + +fn skips_cache(py: Python<'_>, error: &PyErr) -> PyResult { + let database_error: Bound<'_, PyAny> = py.import("sqlite3")?.getattr("DatabaseError")?; + if error.is_instance(py, &database_error) { + return Ok(true); + } + for (module, name) in CACHE_FAULTS { + if error.is_instance(py, &py.import(module)?.getattr(name)?) { + return Ok(true); + } + } + Ok(false) +} + +/// Python's `sqlbuild` distribution version, as the cache key records it. +fn sqlbuild_version(py: Python<'_>) -> PyResult { + let metadata: Bound<'_, PyModule> = py.import("importlib.metadata")?; + match metadata.getattr("version")?.call1(("sqlbuild",)) { + Ok(version) => version.extract(), + Err(error) if error.is_instance(py, &metadata.getattr("PackageNotFoundError")?) => { + Ok(UNKNOWN_VERSION.to_owned()) + } + Err(error) => Err(error), + } +} + +/// The cache key over authored inputs and `cli_vars`, or `None` when they cannot be hashed. +pub(crate) fn lineage_fingerprint( + project_dir: &Bound<'_, PyAny>, + cli_vars: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = project_dir.py(); + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("ensure_ascii", true)?; + options.set_item("separators", (",", ":"))?; + options.set_item("sort_keys", true)?; + let values: Bound<'_, PyAny> = if cli_vars.is_none() { + PyDict::new(py).into_any() + } else { + cli_vars.clone() + }; + let encoded: Bound<'_, PyAny> = match py + .import("json")? + .getattr("dumps")? + .call((values,), Some(&options)) + .and_then(|text| text.call_method1("encode", ("ascii",))) + { + Ok(encoded) => encoded, + Err(error) => { + for name in FINGERPRINT_FAULTS { + if error.is_instance(py, &py.import("builtins")?.getattr(name)?) { + return Ok(None); + } + } + return Err(error); + } + }; + let mut prefix: Vec = CACHE_ALGORITHM_VERSION.as_bytes().to_vec(); + prefix.extend(sqlbuild_version(py)?.as_bytes()); + prefix.extend(encoded.downcast::()?.as_bytes()); + let uncacheable: bool = py.version_info() < FIRST_PYTHON_SKIPPING_INTERRUPTED_LISTINGS; + py.import("sqlbuild._native")? + .getattr("relation_lineage_fingerprint")? + .call1((project_dir, PyBytes::new(py, &prefix), uncacheable))? + .extract() +} + +fn rows<'py, T: pyo3::FromPyObject<'py>>( + connection: &Bound<'py, PyAny>, + sql: &str, +) -> PyResult> { + connection + .call_method1("execute", (sql,))? + .call_method0("fetchall")? + .extract() +} + +fn read_index(connection: &Bound<'_, PyAny>) -> PyResult> { + let resources: Vec = rows( + connection, + "SELECT resource_type, resource_name, relative_path, qualified_name FROM resources", + )?; + let edges: Vec = rows( + connection, + "SELECT upstream_type, upstream_name, downstream_type, downstream_name FROM edges", + )?; + let lookups: Vec = rows( + connection, + "SELECT lookup_name, resource_type, resource_name FROM lookups", + )?; + let tags: Vec = rows( + connection, + "SELECT tag, resource_type, resource_name FROM tags", + )?; + let paths: Vec = rows( + connection, + "SELECT resource_type, resource_name, folder FROM paths", + )?; + if resources.len() + edges.len() + lookups.len() + tags.len() + paths.len() > CACHE_MAX_ROWS { + return Ok(None); + } + let mut nodes: HashMap = HashMap::new(); + for (resource_type, name, relative_path, qualified_name) in resources { + let key: LineageKey = LineageKey { + resource_type, + name, + }; + nodes.insert( + key.clone(), + LineageNode { + key, + relative_path, + qualified_name, + }, + ); + } + let known = |resource_type: String, name: String| -> Option { + let key: LineageKey = LineageKey { + resource_type, + name, + }; + nodes.contains_key(&key).then_some(key) + }; + let mut upstream: Groups = Groups::new(); + let mut downstream: Groups = Groups::new(); + for (upstream_type, upstream_name, downstream_type, downstream_name) in edges { + let (Some(parent), Some(child)) = ( + known(upstream_type, upstream_name), + known(downstream_type, downstream_name), + ) else { + return Ok(None); + }; + upstream.push(child.clone(), parent.clone()); + downstream.push(parent, child); + } + let mut tag_index: Groups = Groups::new(); + for (tag, resource_type, name) in tags { + let Some(key) = known(resource_type, name) else { + return Ok(None); + }; + tag_index.push(tag, key); + } + let mut path_index: Vec<(LineageKey, String)> = Vec::new(); + for (resource_type, name, folder) in paths { + let Some(key) = known(resource_type, name) else { + return Ok(None); + }; + path_index.push((key, folder)); + } + let mut all_keys: Vec<(String, LineageKey)> = Vec::new(); + for (lookup_name, resource_type, name) in lookups { + let Some(key) = known(resource_type, name) else { + return Ok(None); + }; + all_keys.push((lookup_name, key)); + } + Ok(Some(LineageIndex { + nodes, + upstream: upstream.groups, + downstream: downstream.groups, + tags: tag_index.groups, + paths: path_index, + all_keys, + })) +} + +/// Values grouped by key, keys in first-seen order. +struct Groups { + groups: Vec<(K, Vec)>, + positions: HashMap, +} + +impl Groups { + fn new() -> Self { + Self { + groups: Vec::new(), + positions: HashMap::new(), + } + } + + fn push(&mut self, key: K, value: V) { + if let Some(&position) = self.positions.get(&key) { + self.groups[position].1.push(value); + return; + } + self.positions.insert(key.clone(), self.groups.len()); + self.groups.push((key, vec![value])); + } +} + +fn connect<'py>(py: Python<'py>, target: &str, uri: bool) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("timeout", SQLITE_TIMEOUT_SECONDS)?; + if uri { + options.set_item("uri", true)?; + } + py.import("sqlite3")? + .getattr("connect")? + .call((target,), Some(&options)) +} + +/// A verified cached index, or `None` on any cache fault. +pub(crate) fn read_lineage_cache( + py: Python<'_>, + project_dir: &Path, + fingerprint: &str, +) -> PyResult> { + let path: PathBuf = cache_path(project_dir); + match std::fs::metadata(&path) { + Ok(metadata) if metadata.len() <= CACHE_MAX_BYTES => {} + _ => return Ok(None), + } + let read = || -> PyResult> { + let connection: Bound<'_, PyAny> = + connect(py, &format!("file:{}?mode=ro", path.display()), true)?; + let result = (|| -> PyResult> { + let metadata: Vec<(String, String)> = + rows(&connection, "SELECT key, value FROM metadata")?; + let mut metadata: HashMap = metadata.into_iter().collect(); + let expected: [(&str, &str); 3] = [ + ("algorithm_version", CACHE_ALGORITHM_VERSION), + ("input_fingerprint", fingerprint), + ("schema_version", CACHE_SCHEMA_VERSION), + ]; + let matches: bool = metadata.len() == expected.len() + && expected + .iter() + .all(|(key, value)| metadata.remove(*key).as_deref() == Some(*value)); + if !matches { + return Ok(None); + } + read_index(&connection) + })(); + connection.call_method0("close")?; + result + }; + match read() { + Ok(index) => Ok(index), + Err(error) if skips_cache(py, &error)? => Ok(None), + Err(error) => Err(error), + } +} + +fn insert(connection: &Bound<'_, PyAny>, sql: &str, rows: Bound<'_, PyList>) -> PyResult<()> { + connection.call_method1("executemany", (sql, rows))?; + Ok(()) +} + +fn key_parts(key: &LineageKey) -> (&str, &str) { + (key.resource_type.as_str(), key.name.as_str()) +} + +fn write_index( + connection: &Bound<'_, PyAny>, + fingerprint: &str, + index: &LineageIndex, +) -> PyResult<()> { + let py: Python<'_> = connection.py(); + connection.call_method1("executescript", (SCHEMA_SQL,))?; + insert( + connection, + "INSERT INTO metadata (key, value) VALUES (?, ?)", + PyList::new( + py, + [ + ("algorithm_version", CACHE_ALGORITHM_VERSION), + ("input_fingerprint", fingerprint), + ("schema_version", CACHE_SCHEMA_VERSION), + ], + )?, + )?; + let mut resources: Vec> = Vec::new(); + let mut seen: HashSet<&LineageKey> = HashSet::new(); + for (_, key) in &index.all_keys { + if !seen.insert(key) { + continue; + } + let node: Option<&LineageNode> = index.nodes.get(key); + resources.push(PyTuple::new( + py, + [ + Some(key.resource_type.as_str()), + Some(key.name.as_str()), + node.and_then(|node| node.relative_path.as_deref()), + node.and_then(|node| node.qualified_name.as_deref()), + ], + )?); + } + insert( + connection, + "INSERT INTO resources VALUES (?, ?, ?, ?)", + PyList::new(py, resources)?, + )?; + let lookups: Vec<(&str, &str, &str)> = index + .all_keys + .iter() + .map(|(name, key)| (name.as_str(), key.resource_type.as_str(), key.name.as_str())) + .collect(); + insert( + connection, + "INSERT INTO lookups VALUES (?, ?, ?)", + PyList::new(py, lookups)?, + )?; + let mut edges: Vec<(&str, &str, &str, &str)> = Vec::new(); + for (child, parents) in &index.upstream { + for parent in parents { + let ((parent_type, parent_name), (child_type, child_name)) = + (key_parts(parent), key_parts(child)); + edges.push((parent_type, parent_name, child_type, child_name)); + } + } + insert( + connection, + "INSERT INTO edges VALUES (?, ?, ?, ?)", + PyList::new(py, edges)?, + )?; + let mut tags: Vec<(&str, &str, &str)> = Vec::new(); + for (tag, keys) in &index.tags { + for key in keys { + tags.push((tag.as_str(), key.resource_type.as_str(), key.name.as_str())); + } + } + insert( + connection, + "INSERT INTO tags VALUES (?, ?, ?)", + PyList::new(py, tags)?, + )?; + let paths: Vec<(&str, &str, &str)> = index + .paths + .iter() + .map(|(key, folder)| { + ( + key.resource_type.as_str(), + key.name.as_str(), + folder.as_str(), + ) + }) + .collect(); + insert( + connection, + "INSERT INTO paths VALUES (?, ?, ?)", + PyList::new(py, paths)?, + )?; + connection.call_method0("commit")?; + Ok(()) +} + +/// Publish `index` atomically; a failure only skips the cache. +pub(crate) fn write_lineage_cache( + py: Python<'_>, + project_dir: &Path, + fingerprint: &str, + index: &LineageIndex, +) -> PyResult<()> { + let path: PathBuf = cache_path(project_dir); + let Some(directory) = path.parent() else { + return Ok(()); + }; + if std::fs::create_dir_all(directory).is_err() { + return Ok(()); + } + let mut temporary: Option = None; + let mut write = || -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("prefix", ".structural-graph-")?; + options.set_item("dir", directory)?; + let (descriptor, name): (i64, PathBuf) = py + .import("tempfile")? + .getattr("mkstemp")? + .call((), Some(&options))? + .extract()?; + py.import("os")?.getattr("close")?.call1((descriptor,))?; + temporary = Some(name.clone()); + let connection: Bound<'_, PyAny> = connect(py, &name.to_string_lossy(), false)?; + let written: PyResult<()> = write_index(&connection, fingerprint, index); + connection.call_method0("close")?; + written?; + std::fs::rename(&name, &path)?; + Ok(()) + }; + let outcome: PyResult<()> = write(); + if let Some(name) = temporary { + let _ = std::fs::remove_file(name); + } + match outcome { + Err(error) if !skips_cache(py, &error)? => Err(error), + _ => Ok(()), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs new file mode 100644 index 0000000000..a1fdf1b70f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -0,0 +1,724 @@ +//! `sqb lineage`: load or build the relation graph, select lineage, and print it. + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::lineage_command::constants::{ + BOTH_DIRECTIONS, COLUMN_TARGET_SEPARATOR, DOWNSTREAM_DIRECTION, MODEL_RESOURCE_TYPE, + RICH_STATUS_MODEL_THRESHOLD, SELECTOR_INTERSECTION_MARKER, UPSTREAM_DIRECTION, +}; +use sqlbuild_analysis::lineage_command::errors::LineageError; +use sqlbuild_analysis::lineage_command::main::column_candidates::column_candidates; +use sqlbuild_analysis::lineage_command::main::lineage_path_nodes::lineage_path_nodes; +use sqlbuild_analysis::lineage_command::main::lineage_stdout::lineage_stdout; +use sqlbuild_analysis::lineage_command::main::normalize_lineage_target::normalize_lineage_target; +use sqlbuild_analysis::lineage_command::main::parse_lineage_depth::parse_lineage_depth; +use sqlbuild_analysis::lineage_command::main::render_column_trace::render_column_trace; +use sqlbuild_analysis::lineage_command::main::render_lineage_graph::render_lineage_graph; +use sqlbuild_analysis::lineage_command::main::select_anchored_lineage::select_anchored_lineage; +use sqlbuild_analysis::lineage_command::main::select_directional_lineage::select_directional_lineage; +use sqlbuild_analysis::lineage_command::main::select_target_lineage::select_target_lineage; +use sqlbuild_analysis::lineage_command::main::validate_lineage_request::validate_lineage_request; +use sqlbuild_analysis::lineage_command::models::{ + ColumnCandidates, ColumnTrace, LineageAnchors, LineageColumn, LineageColumnEdge, LineageIndex, + LineageKey, LineageRendering, LineageRequestShape, SemanticUse, +}; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::lineage::lineage_cache::{ + lineage_fingerprint, read_lineage_cache, write_lineage_cache, +}; +use crate::bindings::_helpers::lineage::lineage_index::{graph_index, index_views, lineage_keys}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const STATUS_MODULE: &str = "sqlbuild.presentation.main._maybe_status"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; +const SELECTION_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selection"; +const SELECTOR_TOKENS_MODULE: &str = "sqlbuild.compiler.planner.main.selection._selector_tokens"; +const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; +const SELECTOR_MATCH_MODULE: &str = "sqlbuild.compiler.planner.main.selection._selector_match"; +const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +const UNKNOWN_SELECTOR_ERROR_CODE: &str = "S007"; +const RICH_MODE: &str = "rich"; +const ANALYSIS_HELP: &str = "enable SQL analysis or install SQLBuild with Polyglot support"; +/// Selector views every planner selector function reads. +const GRAPH_VIEW_NAMES: [&str; 5] = [ + "all_keys", + "upstream_deps", + "downstream_deps", + "tag_index", + "path_index", +]; +/// Selector kinds that match a set of resources rather than a named anchor. +const SET_SELECTOR_KINDS: [&str; 2] = ["tag", "path"]; + +/// The relation graph lineage reads: a cached index, or a compiled graph with its adapter. +enum LineageSource<'py> { + Cached(LineageIndex), + Compiled { + index: LineageIndex, + graph: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + }, +} + +impl LineageSource<'_> { + fn index(&self) -> &LineageIndex { + match self { + Self::Cached(index) | Self::Compiled { index, .. } => index, + } + } +} + +fn keywords<'py>( + py: Python<'py>, + items: &[(&str, &Bound<'py, PyAny>)], +) -> PyResult> { + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in items { + dict.set_item(*name, value)?; + } + Ok(dict) +} + +fn imported<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { + py.import(module)?.getattr(name) +} + +/// `CliUserError(message, code=..., help=...)`. +fn user_error(py: Python<'_>, error: LineageError) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", error.code)?; + options.set_item("help", error.help)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((error.message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn strings(value: &Bound<'_, PyAny>) -> PyResult> { + value.try_iter()?.map(|item| item?.extract()).collect() +} + +fn name_lookup(index: &LineageIndex) -> HashMap<&str, &LineageKey> { + index + .all_keys + .iter() + .map(|(name, key)| (name.as_str(), key)) + .collect() +} + +fn normalized_targets( + py: Python<'_>, + index: &LineageIndex, + targets: &[String], +) -> PyResult> { + let names: HashMap<&str, &LineageKey> = name_lookup(index); + let lookup = |name: &str| names.get(name).map(|key| (*key).clone()); + targets + .iter() + .map(|target| { + normalize_lineage_target(target, &lookup).map_err(|error| user_error(py, error)) + }) + .collect() +} + +/// `discover_project_inputs` with lineage's analysis override. +fn discover<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, + analysis_override: Option, +) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("project_dir", project_dir)?; + options.set_item("sql_analysis_enabled_override", analysis_override)?; + options.set_item("extract_output_column_locations", false)?; + imported(py, NATIVE_MODULE, "discover_project_inputs")?.call((), Some(&options)) +} + +fn reject_unit_tests( + py: Python<'_>, + request: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, +) -> PyResult<()> { + let options: Bound<'_, PyDict> = keywords( + py, + &[ + ("select", &request.getattr("select")?), + ("exclude", &request.getattr("exclude")?), + ("discovered_inputs", discovered), + ], + )?; + imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")? + .call((), Some(&options))?; + Ok(()) +} + +/// The cached relation index, or a freshly built graph that refreshes the cache. +fn prepare_source<'py>( + py: Python<'py>, + request: &Bound<'py, PyAny>, + (project_dir, compiled): (&Bound<'py, PyAny>, bool), +) -> PyResult> { + let directory: PathBuf = project_dir.extract()?; + let fingerprint: Option = if compiled { + None + } else { + lineage_fingerprint(project_dir, &request.getattr("cli_vars")?)? + }; + if let Some(fingerprint) = &fingerprint + && let Some(index) = read_lineage_cache(py, &directory, fingerprint)? + { + return Ok(LineageSource::Cached(index)); + } + let discovered: Bound<'py, PyAny> = + discover(py, project_dir, if compiled { None } else { Some(false) })?; + reject_unit_tests(py, request, &discovered)?; + let adapter_name: Bound<'py, PyAny> = + imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?.call( + (), + Some(&keywords( + py, + &[ + ("project_config", &discovered.getattr("project_config")?), + ("local_config", &discovered.getattr("local_config")?), + ], + )?), + )?; + let adapter: Bound<'py, PyAny> = imported(py, ADAPTERS_MODULE, "resolve_adapter")?.call( + (), + Some(&keywords( + py, + &[ + ("adapter_name", &adapter_name), + ("project_dir", project_dir), + ], + )?), + )?; + let no_validation: bool = request.getattr("no_sql_validation")?.extract::()? || !compiled; + let options: Bound<'py, PyDict> = keywords( + py, + &[ + ("discovered_inputs", &discovered), + ("adapter", &adapter), + ("cli_vars", &request.getattr("cli_vars")?), + ], + )?; + options.set_item("no_sql_validation", no_validation)?; + options.set_item("skip_column_inference", !compiled)?; + let graph: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "build_project_graph")?.call((), Some(&options))?; + let index: LineageIndex = graph_index(&graph)?; + if let Some(fingerprint) = &fingerprint { + write_lineage_cache(py, &directory, fingerprint, &index)?; + return Ok(LineageSource::Cached(index)); + } + Ok(LineageSource::Compiled { + index, + graph, + adapter, + }) +} + +fn compiled_graph_error(py: Python<'_>, message: &str, help: bool) -> PyErr { + let error: LineageError = LineageError::new("C307", message); + user_error( + py, + if help { + error.with_help(ANALYSIS_HELP) + } else { + error + }, + ) +} + +fn semantic_use(value: &Bound<'_, PyAny>) -> PyResult { + let source: Bound<'_, PyAny> = value.getattr("source")?; + Ok(SemanticUse { + consumer_model: value.getattr("consumer_model")?.str()?.extract()?, + context: value.getattr("context")?.str()?.extract()?, + expression: value.getattr("expression_sql")?.str()?.extract()?, + source: lineage_column(&source)?, + confidence: value.getattr("confidence")?.getattr("value")?.extract()?, + line: value.getattr("line")?.extract()?, + column: value.getattr("column")?.extract()?, + }) +} + +/// Direct semantic column uses of the targeted or selected models. +fn semantic_uses( + py: Python<'_>, + request: &Bound<'_, PyAny>, + source: &LineageSource<'_>, +) -> PyResult> { + let LineageSource::Compiled { + index, + graph, + adapter, + } = source + else { + return Err(compiled_graph_error( + py, + "semantic uses require a compiled project graph", + false, + )); + }; + let targets: Vec = strings(&request.getattr("targets")?)?; + let model_names: Vec = if targets.is_empty() { + let options: Bound<'_, PyDict> = keywords( + py, + &[ + ("graph", graph), + ("select", &request.getattr("select")?), + ("exclude", &request.getattr("exclude")?), + ], + )?; + let keys: Vec = lineage_keys( + &imported(py, GRAPH_SELECTION_MODULE, "select_project_graph")? + .call((), Some(&options))?, + )?; + keys.into_iter() + .filter(|key| key.resource_type == MODEL_RESOURCE_TYPE) + .map(|key| key.name) + .collect() + } else { + normalized_targets(py, index, &targets)? + .into_iter() + .map(|target| { + target + .split_once(COLUMN_TARGET_SEPARATOR) + .map_or(target.clone(), |(model, _)| model.to_owned()) + }) + .collect() + }; + let uses: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "build_direct_semantic_uses")?.call1(( + graph.getattr("project")?, + adapter.call_method0("sql_analysis_dialect")?, + PyFrozenSet::new(py, model_names)?, + ))?; + uses.try_iter()? + .map(|value| semantic_use(&value?)) + .collect() +} + +fn lineage_column(column: &Bound<'_, PyAny>) -> PyResult { + Ok(LineageColumn { + resource_type: column.getattr("resource_type")?.str()?.extract()?, + resource_name: column.getattr("resource_name")?.extract()?, + column_name: column.getattr("column_name")?.extract()?, + }) +} + +fn column_edge(edge: &Bound<'_, PyAny>) -> PyResult { + Ok(LineageColumnEdge { + source: lineage_column(&edge.getattr("source")?)?, + target: lineage_column(&edge.getattr("target")?)?, + transform: edge.getattr("transform_kind")?.str()?.extract()?, + confidence: edge.getattr("confidence")?.str()?.extract()?, + }) +} + +/// Edges reached depth first from one column, as Python's stack walk visits them. +fn traced_edges( + lineage: &Bound<'_, PyAny>, + (resource_name, column_name): (&str, &str), + (direction, max_depth): (&str, Option), +) -> PyResult> { + let mut result: Vec = Vec::new(); + if max_depth == Some(0) { + return Ok(result); + } + let downstream: bool = direction == DOWNSTREAM_DIRECTION; + let mut stack: Vec<(String, String, i64)> = + vec![(resource_name.to_owned(), column_name.to_owned(), 0)]; + let mut visited: HashSet<(String, String)> = HashSet::new(); + while let Some((resource, column, depth)) = stack.pop() { + if !visited.insert((resource.clone(), column.clone())) { + continue; + } + if max_depth.is_some_and(|maximum| depth >= maximum) { + continue; + } + let edges: Bound<'_, PyAny> = if downstream { + let options: Bound<'_, PyDict> = PyDict::new(lineage.py()); + options.set_item("resource_name", &resource)?; + options.set_item("column_name", &column)?; + lineage.call_method("column_consumers", (), Some(&options))? + } else { + lineage.call_method1("edges_targeting", (&resource,))? + }; + for edge in edges.try_iter()? { + let edge: LineageColumnEdge = column_edge(&edge?)?; + let next: &LineageColumn = if downstream { + &edge.target + } else { + if edge.target.column_name != column { + continue; + } + &edge.source + }; + stack.push(( + next.resource_name.clone(), + next.column_name.clone(), + depth + 1, + )); + result.push(edge); + } + } + Ok(result) +} + +/// `with maybe_status(...)` around building column lineage. +fn column_lineage<'py>( + py: Python<'py>, + (graph, mode): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + candidates: &ColumnCandidates, +) -> PyResult> { + let count: usize = candidates.model_names.len(); + let rich: bool = mode.str()?.extract::()? == RICH_MODE; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item( + "message", + format!("Analyzing column lineage for {count} models..."), + )?; + options.set_item("enabled", rich && count >= RICH_STATUS_MODEL_THRESHOLD)?; + let status: Bound<'py, PyAny> = + imported(py, STATUS_MODULE, "maybe_status")?.call((), Some(&options))?; + status.call_method0("__enter__")?; + let built: PyResult> = + imported(py, NATIVE_MODULE, "build_project_column_lineage").and_then(|build| { + build.call1(( + graph.getattr("project")?, + py.None(), + mode, + PyFrozenSet::new(py, &candidates.model_names)?, + )) + }); + status.call_method1("__exit__", (py.None(), py.None(), py.None()))?; + built +} + +/// The column trace of a `model.column` target, or `None` when the target is not a model column. +fn column_trace( + py: Python<'_>, + source: &LineageSource<'_>, + (target, mode): (&str, &Bound<'_, PyAny>), + (direction, depth): (&str, Option), +) -> PyResult> { + let Some((resource_name, column_name)) = target.rsplit_once(COLUMN_TARGET_SEPARATOR) else { + return Ok(None); + }; + let LineageSource::Compiled { index, graph, .. } = source else { + return Err(compiled_graph_error( + py, + "column lineage requires a compiled project graph", + true, + )); + }; + let Some(key) = name_lookup(index) + .get(resource_name) + .map(|key| (*key).clone()) + else { + return Ok(None); + }; + if key.resource_type != MODEL_RESOURCE_TYPE { + return Ok(None); + } + if direction == BOTH_DIRECTIONS { + return Err(user_error( + py, + LineageError::new( + "C306", + "column lineage supports --direction upstream or downstream, not both", + ), + )); + } + let candidates: ColumnCandidates = column_candidates(index, &key, (direction, depth)); + let lineage: Bound<'_, PyAny> = column_lineage(py, (graph, mode), &candidates)?; + if lineage.is_none() { + return Err(compiled_graph_error( + py, + "column lineage requires SQL analysis to be enabled and available", + true, + )); + } + Ok(Some(ColumnTrace { + target: LineageColumn { + resource_type: key.resource_type.clone(), + resource_name: key.name.clone(), + column_name: column_name.to_owned(), + }, + trace: traced_edges(&lineage, (&key.name, column_name), (direction, depth))?, + direction: direction.to_owned(), + mode: mode.getattr("value")?.extract()?, + max_depth: depth, + analyzed_model_count: candidates.model_names.len(), + truncated: candidates.truncated, + })) +} + +/// Selector views of the source: the compiled graph's own dicts, or dicts built from the index. +fn selector_views<'py>( + py: Python<'py>, + source: &LineageSource<'py>, +) -> PyResult> { + match source { + LineageSource::Compiled { graph, .. } => graph.getattr("lineage_views"), + LineageSource::Cached(index) => index_views(py, index), + } +} + +fn view_options<'py>(views: &Bound<'py, PyAny>, names: &[&str]) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(views.py()); + for name in names { + options.set_item(*name, views.getattr(*name)?)?; + } + Ok(options) +} + +/// Depth anchors of selectors the shared resolver already validated. +fn selector_anchors( + py: Python<'_>, + (views, index): (&Bound<'_, PyAny>, &LineageIndex), + select: &[String], + require_clear_anchors: bool, +) -> PyResult { + let parse: Bound<'_, PyAny> = imported(py, SELECTOR_PARSE_MODULE, "parse_project_selector")?; + let matcher: Bound<'_, PyAny> = imported(py, SELECTOR_MATCH_MODULE, "match_project_selector")?; + let path_class: Bound<'_, PyAny> = imported(py, PLANNER_MODELS, "PathSelector")?; + let names: HashMap<&str, &LineageKey> = name_lookup(index); + let named = |name: String| -> PyResult { + names + .get(name.as_str()) + .map(|key| (*key).clone()) + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(name)) + }; + let mut anchors: LineageAnchors = LineageAnchors::default(); + for raw in select { + for token in raw.split_whitespace() { + if token.contains(SELECTOR_INTERSECTION_MARKER) && require_clear_anchors { + return Err(user_error( + py, + LineageError::new( + "C308", + "--depth cannot be combined with comma-intersection selectors", + ), + )); + } + for part in token.split(SELECTOR_INTERSECTION_MARKER) { + let parsed: Bound<'_, PyAny> = parse.call1((part,))?; + if parsed.is_instance(&path_class)? { + let start: LineageKey = named(parsed.getattr("start_name")?.extract()?)?; + let end: LineageKey = named(parsed.getattr("end_name")?.extract()?)?; + anchors + .retained + .extend(lineage_path_nodes(index, &start, &end)); + anchors.upstream.push(start); + anchors.downstream.push(end); + continue; + } + let kind: String = parsed.getattr("kind")?.str()?.extract()?; + let set_kind: bool = SET_SELECTOR_KINDS.contains(&kind.as_str()); + if set_kind && require_clear_anchors { + return Err(user_error( + py, + LineageError::new( + "C309", + "--depth requires name, source, seed, or path-between selectors", + ), + )); + } + let options: Bound<'_, PyDict> = + view_options(views, &["all_keys", "tag_index", "path_index"])?; + options.set_item("parsed", &parsed)?; + let matched: Vec = lineage_keys(&matcher.call((), Some(&options))?)?; + let upstream: bool = parsed.getattr("upstream")?.extract()?; + let downstream: bool = parsed.getattr("downstream")?.extract()?; + if set_kind || upstream { + anchors.upstream.extend(matched.iter().cloned()); + } + if set_kind || downstream { + anchors.downstream.extend(matched.iter().cloned()); + } + if !set_kind && !upstream && !downstream { + anchors.retained.extend(matched); + } + } + } + } + Ok(anchors) +} + +/// Lineage of `--select`/`--exclude`, expanded by direction or trimmed to depth. +fn selector_rendering( + py: Python<'_>, + request: &Bound<'_, PyAny>, + source: &LineageSource<'_>, + (depth, style): (Option, CliStyle), +) -> PyResult { + let views: Bound<'_, PyAny> = selector_views(py, source)?; + let select: Bound<'_, PyAny> = request.getattr("select")?; + let exclude: Bound<'_, PyAny> = request.getattr("exclude")?; + let options: Bound<'_, PyDict> = view_options(&views, &GRAPH_VIEW_NAMES)?; + options.set_item("select", &select)?; + options.set_item("exclude", &exclude)?; + let selected: Vec = lineage_keys( + &imported(py, SELECTION_MODULE, "resolve_project_selectors")?.call((), Some(&options))?, + )?; + let index: &LineageIndex = source.index(); + let direction: Option = request.getattr("direction")?.extract()?; + let output_format: String = request.getattr("output_format")?.extract()?; + let graph = if let Some(direction) = direction { + let options: Bound<'_, PyDict> = view_options(&views, &GRAPH_VIEW_NAMES)?; + options.set_item("selectors", &exclude)?; + let excluded: Vec = lineage_keys( + &imported( + py, + SELECTOR_TOKENS_MODULE, + "resolve_project_selector_tokens", + )? + .call((), Some(&options))?, + )?; + select_directional_lineage(index, (&selected, &excluded), (&direction, depth)) + } else { + let anchors: LineageAnchors = + selector_anchors(py, (&views, index), &strings(&select)?, depth.is_some())?; + select_anchored_lineage(index, &selected, (&anchors, depth)) + }; + Ok(render_lineage_graph(&graph, &output_format, style)) +} + +/// Re-raise an unknown selector from a cached graph as the unit-test error it may hide. +fn selector_failure( + py: Python<'_>, + request: &Bound<'_, PyAny>, + (source, project_dir): (&LineageSource<'_>, &Bound<'_, PyAny>), + error: PyErr, +) -> PyErr { + let check = || -> PyResult<()> { + let planner_error: Bound<'_, PyAny> = + imported(py, PLANNER_EXCEPTIONS, "PlannerInputError")?; + if !matches!(source, LineageSource::Cached(_)) + || !error.is_instance(py, &planner_error) + || error.value(py).getattr("code")?.extract::()? != UNKNOWN_SELECTOR_ERROR_CODE + { + return Ok(()); + } + reject_unit_tests(py, request, &discover(py, project_dir, Some(false))?) + }; + match check() { + Ok(()) => error, + Err(raised) => raised, + } +} + +fn rendering( + py: Python<'_>, + request: &Bound<'_, PyAny>, + (source, project_dir): (&LineageSource<'_>, &Bound<'_, PyAny>), + (depth, style): (Option, CliStyle), +) -> PyResult { + let targets: Vec = strings(&request.getattr("targets")?)?; + let output_format: String = request.getattr("output_format")?.extract()?; + if targets.is_empty() { + return selector_rendering(py, request, source, (depth, style)) + .map_err(|error| selector_failure(py, request, (source, project_dir), error)); + } + let targets: Vec = normalized_targets(py, source.index(), &targets)?; + let direction: String = request + .getattr("direction")? + .extract::>()? + .unwrap_or_else(|| UPSTREAM_DIRECTION.to_owned()); + if let [target] = targets.as_slice() + && let Some(trace) = column_trace( + py, + source, + (target, &request.getattr("lineage_mode")?), + (&direction, depth), + )? + { + return Ok(render_column_trace(&trace, &output_format, style)); + } + let graph = select_target_lineage(source.index(), &targets, (&direction, depth)) + .map_err(|error| user_error(py, error))?; + Ok(render_lineage_graph(&graph, &output_format, style)) +} + +fn request_shape(request: &Bound<'_, PyAny>) -> PyResult { + Ok(LineageRequestShape { + targets: strings(&request.getattr("targets")?)?, + select: strings(&request.getattr("select")?)?, + exclude: strings(&request.getattr("exclude")?)?, + }) +} + +/// Run one `sqb lineage` request and print its result. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_lineage_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let shape: LineageRequestShape = request_shape(request)?; + validate_lineage_request(&shape).map_err(|error| user_error(py, error))?; + let depth: Option = parse_lineage_depth(&request.getattr("depth")?.extract::()?) + .map_err(|error| user_error(py, error))?; + let project_dir: Bound<'_, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, + }; + let include_uses: bool = request.getattr("include_uses")?.extract()?; + let compiled: bool = include_uses + || shape + .targets + .iter() + .any(|target| target.contains(COLUMN_TARGET_SEPARATOR)); + let source: LineageSource<'_> = prepare_source(py, request, (&project_dir, compiled))?; + let uses: Option> = if include_uses { + Some(semantic_uses(py, request, &source)?) + } else { + None + }; + let style: CliStyle = CliStyle { + use_color: imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .extract()?, + }; + let rendered: LineageRendering = + rendering(py, request, (&source, &project_dir), (depth, style))?; + let output: String = lineage_stdout(rendered, uses.as_deref()) + .map_err(|error| PyValueError::new_err(format!("{error:?}")))?; + py.import("sys")? + .getattr("stdout")? + .call_method1("write", (output,))?; + Ok(0) +} + +/// The relation lineage cache key, or `None` when the inputs cannot be hashed. +#[pyfunction] +#[pyo3(signature = (*, project_dir, cli_vars))] +fn lineage_cache_fingerprint( + project_dir: &Bound<'_, PyAny>, + cli_vars: &Bound<'_, PyAny>, +) -> PyResult> { + lineage_fingerprint(project_dir, cli_vars) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_lineage_command, module)?)?; + module.add_function(wrap_pyfunction!(lineage_cache_fingerprint, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs new file mode 100644 index 0000000000..3ddf618a35 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs @@ -0,0 +1,218 @@ +//! The relation lineage index read from a project graph, and its Python selector views. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use sqlbuild_analysis::lineage_command::models::{LineageIndex, LineageKey, LineageNode}; + +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const GRAPH_MODELS: &str = "sqlbuild.compiler.graph.models"; + +/// A Python `CompiledObjectKey` as a lineage key. +pub(crate) fn lineage_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok(LineageKey { + resource_type: key.getattr("resource_type")?.str()?.extract()?, + name: key.getattr("name")?.extract()?, + }) +} + +/// Lineage keys of every Python key in `keys`. +pub(crate) fn lineage_keys(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| lineage_key(&key?)).collect() +} + +/// Python `CompiledObjectKey`s for lineage keys, one object per key. +pub(crate) struct PythonKeys<'py> { + class: Bound<'py, PyAny>, + made: HashMap>, +} + +impl<'py> PythonKeys<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + Ok(Self { + class: py.import(COMPILE_MODELS)?.getattr("CompiledObjectKey")?, + made: HashMap::new(), + }) + } + + pub(crate) fn key(&mut self, key: &LineageKey) -> PyResult> { + if let Some(made) = self.made.get(key) { + return Ok(made.clone()); + } + let made: Bound<'py, PyAny> = self + .class + .call1((key.resource_type.as_str(), key.name.as_str()))?; + self.made.insert(key.clone(), made.clone()); + Ok(made) + } + + fn tuple(&mut self, keys: &[LineageKey]) -> PyResult> { + let items: Vec> = keys + .iter() + .map(|key| self.key(key)) + .collect::>()?; + PyTuple::new(self.class.py(), items) + } +} + +fn ordered_edges(edges: &Bound<'_, PyAny>) -> PyResult)>> { + let mut ordered: Vec<(LineageKey, Vec)> = Vec::new(); + for item in edges.call_method0("items")?.try_iter()? { + let (key, targets): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + ordered.push((lineage_key(&key)?, lineage_keys(&targets)?)); + } + Ok(ordered) +} + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +/// A source entry's `database.schema.table`, as far as it is set. +fn source_relation_name(entry: &Bound<'_, PyAny>) -> PyResult> { + let part = |name: &str| -> PyResult> { + match entry.getattr(name) { + Ok(value) if !value.is_none() => Ok(Some(value.extract()?)), + _ => Ok(None), + } + }; + let Some(table) = part("table")? else { + return Ok(None); + }; + Ok(Some(match (part("database")?, part("schema")?) { + (Some(database), Some(schema)) => format!("{database}.{schema}.{table}"), + (_, Some(schema)) => format!("{schema}.{table}"), + _ => table, + })) +} + +/// Display facts of every model, seed and source, models first as Python looks them up. +fn project_nodes(project: &Bound<'_, PyAny>) -> PyResult> { + let mut nodes: HashMap = HashMap::new(); + let mut add = |node: LineageNode| { + nodes.entry(node.key.clone()).or_insert(node); + }; + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + add(LineageNode { + key: lineage_key(&model.getattr("key")?)?, + relative_path: optional_text(model.getattr("relative_path")?)?, + qualified_name: optional_text( + model.getattr("destination")?.getattr("qualified_name")?, + )?, + }); + } + for seed in project.getattr("seeds")?.try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + add(LineageNode { + key: lineage_key(&seed.getattr("key")?)?, + relative_path: optional_text(seed.getattr("seed_file")?.getattr("relative_path")?)?, + qualified_name: optional_text(seed.getattr("destination")?.getattr("qualified_name")?)?, + }); + } + for source in project.getattr("sources")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + add(LineageNode { + key: lineage_key(&source.getattr("key")?)?, + relative_path: optional_text(source.getattr("source_file")?.getattr("relative_path")?)?, + qualified_name: source_relation_name(&source.getattr("source_entry")?)?, + }); + } + Ok(nodes) +} + +/// The lineage index of a compiled `ProjectGraph`; every named key has a node. +pub(crate) fn graph_index(graph: &Bound<'_, PyAny>) -> PyResult { + let views: Bound<'_, PyAny> = graph.getattr("lineage_views")?; + let mut all_keys: Vec<(String, LineageKey)> = Vec::new(); + for item in views + .getattr("all_keys")? + .call_method0("items")? + .try_iter()? + { + let (name, key): (String, Bound<'_, PyAny>) = item?.extract()?; + all_keys.push((name, lineage_key(&key)?)); + } + let mut tags: Vec<(String, Vec)> = Vec::new(); + for item in views + .getattr("tag_index")? + .call_method0("items")? + .try_iter()? + { + let (tag, keys): (String, Bound<'_, PyAny>) = item?.extract()?; + tags.push((tag, lineage_keys(&keys)?)); + } + let mut paths: Vec<(LineageKey, String)> = Vec::new(); + for item in views + .getattr("path_index")? + .call_method0("items")? + .try_iter()? + { + let (key, folder): (Bound<'_, PyAny>, String) = item?.extract()?; + paths.push((lineage_key(&key)?, folder)); + } + let mut nodes: HashMap = project_nodes(&graph.getattr("project")?)?; + for (_, key) in &all_keys { + nodes.entry(key.clone()).or_insert_with(|| LineageNode { + key: key.clone(), + relative_path: None, + qualified_name: None, + }); + } + Ok(LineageIndex { + nodes, + upstream: ordered_edges(&views.getattr("upstream_deps")?)?, + downstream: ordered_edges(&views.getattr("downstream_deps")?)?, + tags, + paths, + all_keys, + }) +} + +/// `LineageGraphViews` of an index, for the planner's selector functions. +pub(crate) fn index_views<'py>( + py: Python<'py>, + index: &LineageIndex, +) -> PyResult> { + let mut keys: PythonKeys<'py> = PythonKeys::new(py)?; + let edges = |pairs: &[(LineageKey, Vec)], + keys: &mut PythonKeys<'py>| + -> PyResult> { + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (key, targets) in pairs { + dict.set_item(keys.key(key)?, keys.tuple(targets)?)?; + } + Ok(dict) + }; + let upstream: Bound<'py, PyDict> = edges(&index.upstream, &mut keys)?; + let downstream: Bound<'py, PyDict> = edges(&index.downstream, &mut keys)?; + let tags: Bound<'py, PyDict> = PyDict::new(py); + for (tag, tagged) in &index.tags { + let items: Vec> = tagged + .iter() + .map(|key| keys.key(key)) + .collect::>()?; + tags.set_item(tag, PyFrozenSet::new(py, items)?)?; + } + let paths: Bound<'py, PyDict> = PyDict::new(py); + for (key, folder) in &index.paths { + paths.set_item(keys.key(key)?, folder)?; + } + let names: Bound<'py, PyDict> = PyDict::new(py); + for (name, key) in &index.all_keys { + names.set_item(name, keys.key(key)?)?; + } + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("upstream_deps", upstream)?; + options.set_item("downstream_deps", downstream)?; + options.set_item("tag_index", tags)?; + options.set_item("path_index", paths)?; + options.set_item("all_keys", names)?; + py.import(GRAPH_MODELS)? + .getattr("LineageGraphViews")? + .call((), Some(&options)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs index cb9ea47c07..26032298d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs @@ -1,7 +1,9 @@ pub(crate) mod column_lineage_graph; +pub(crate) mod lineage_cache; +pub(crate) mod lineage_command; +pub(crate) mod lineage_index; pub(crate) mod lineage_objects; pub(crate) mod project_lineage; pub(crate) mod project_schema; pub(crate) mod registration; pub(crate) mod relation_fingerprint; -pub(crate) mod tree_render; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs index cb84372d9d..f624ff31f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs @@ -1,14 +1,14 @@ -//! Register the native column lineage functions and classes. +//! Register the native column lineage functions and classes, and `sqb lineage`. use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::lineage::{ - column_lineage_graph, project_lineage, relation_fingerprint, tree_render, + column_lineage_graph, lineage_command, project_lineage, relation_fingerprint, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { column_lineage_graph::register(module)?; project_lineage::register(module)?; relation_fingerprint::register(module)?; - tree_render::register(module) + lineage_command::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs deleted file mode 100644 index 724458e623..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs +++ /dev/null @@ -1,265 +0,0 @@ -//! Lineage output: box-drawing dependency and column-trace trees, and column JSON payloads. - -use std::collections::HashSet; - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::lineage_graph::main::trace_limit_note::trace_limit_note; -use sqlbuild_analysis::lineage_graph::main::tree_lines::tree_lines; -use sqlbuild_analysis::lineage_graph::models::TreeLine; -use sqlbuild_analysis::lineage_graph::types::TreeVisitor; - -use crate::bindings::_helpers::boundary::keyword_request::Request; - -const BUILTINS_MODULE: &str = "builtins"; - -/// Box-drawing dependency tree lines for one focus node, expanding each node once. -#[pyfunction] -#[pyo3(signature = (**request))] -fn render_dependency_tree(request: Option<&Bound<'_, PyDict>>) -> PyResult> { - let request = Request::new("render_dependency_tree", request)?; - let py: Python<'_> = request.py(); - let mut tree = DependencyTree { - deps: request.get("deps")?, - sort_key: request.get("sort_key")?, - format_node: request.get("format_node")?, - branch_style: request.get("branch_style")?, - already_shown: request.get("already_shown")?, - sorted: py.import(BUILTINS_MODULE)?.getattr("sorted")?, - identities: PyDict::new(py), - lines: Vec::new(), - }; - let focus: Bound<'_, PyAny> = request.get("focus")?; - let seen: Bound<'_, PyAny> = request.get("seen")?; - let roots: Vec> = tree.children(&focus)?; - let mut shown: HashSet = HashSet::new(); - for node in seen.try_iter()? { - let _ = shown.insert(tree.key(&node?)?); - } - tree_lines(roots, shown, &mut tree)?; - Ok(tree.lines) -} - -/// Box-drawing column-trace tree lines plus any truncation note. -#[pyfunction] -#[pyo3(signature = (**request))] -fn render_column_trace_tree(request: Option<&Bound<'_, PyDict>>) -> PyResult> { - let request = Request::new("render_column_trace_tree", request)?; - let target: Bound<'_, PyAny> = request.get("target")?; - let mut tree = ColumnTraceTree { - deps: request.get("deps")?, - column_id: request.get("column_id")?, - related_column: request.get("related_column")?, - format_related: request.get("format_related")?, - branch_style: request.get("branch_style")?, - already_shown: request.get("already_shown")?, - last_key: String::new(), - lines: Vec::new(), - }; - let seen: HashSet = HashSet::from([text(&tree.column_id.call1((&target,))?)?]); - let roots: Vec> = - tree.sorted_edges(&text(&tree.column_id.call1((&target,))?)?)?; - tree_lines(roots, seen, &mut tree)?; - let mut lines: Vec = tree.lines; - lines.extend(limit_note( - &request.get("total")?, - &request.get("limit")?, - &request.get("note_style")?, - )?); - Ok(lines) -} - -/// The shared truncation note when a column trace is limited. -#[pyfunction] -#[pyo3(signature = (**request))] -fn render_column_trace_limit_note(request: Option<&Bound<'_, PyDict>>) -> PyResult> { - let request = Request::new("render_column_trace_limit_note", request)?; - limit_note( - &request.get("total")?, - &request.get("limit")?, - &request.get("note_style")?, - ) -} - -/// A qualified lineage column's JSON payload, its resource type rendered by the caller. -#[pyfunction] -#[pyo3(signature = (**request))] -fn serialize_column<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { - let request = Request::new("serialize_column", request)?; - column_payload( - &request.get("column")?, - &request.get("render_resource_type")?, - ) -} - -/// A column lineage edge's JSON payload, resource types rendered by the caller. -#[pyfunction] -#[pyo3(signature = (**request))] -fn serialize_column_edge<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("serialize_column_edge", request)?; - let edge: Bound<'py, PyAny> = request.get("edge")?; - let render: Bound<'py, PyAny> = request.get("render_resource_type")?; - let payload: Bound<'py, PyDict> = PyDict::new(request.py()); - payload.set_item("source", column_payload(&edge.getattr("source")?, &render)?)?; - payload.set_item("target", column_payload(&edge.getattr("target")?, &render)?)?; - payload.set_item("transform", edge.getattr("transform_kind")?.str()?)?; - payload.set_item("confidence", edge.getattr("confidence")?.str()?)?; - Ok(payload) -} - -/// `str(value)`, as an f-string renders a callback's text. -fn text(value: &Bound<'_, PyAny>) -> PyResult { - value.str()?.extract() -} - -fn limit_note( - total: &Bound<'_, PyAny>, - limit: &Bound<'_, PyAny>, - note_style: &Bound<'_, PyAny>, -) -> PyResult> { - let notes: Vec = trace_limit_note(total.extract()?, limit.extract()?); - if notes.is_empty() { - return Ok(notes); - } - let mut lines: Vec = vec![String::new()]; - for note in notes { - lines.push(note_style.call1((note,))?.extract()?); - } - Ok(lines) -} - -fn column_payload<'py>( - column: &Bound<'py, PyAny>, - render_resource_type: &Bound<'py, PyAny>, -) -> PyResult> { - let payload: Bound<'py, PyDict> = PyDict::new(column.py()); - payload.set_item("resource_type", render_resource_type.call1((column,))?)?; - payload.set_item("resource_name", column.getattr("resource_name")?)?; - payload.set_item("column_name", column.getattr("column_name")?)?; - Ok(payload) -} - -/// Dependency nodes shown once by Python equality, children sorted by the caller's key. -struct DependencyTree<'py> { - deps: Bound<'py, PyAny>, - sort_key: Bound<'py, PyAny>, - format_node: Bound<'py, PyAny>, - branch_style: Bound<'py, PyAny>, - already_shown: Bound<'py, PyAny>, - sorted: Bound<'py, PyAny>, - identities: Bound<'py, PyDict>, - lines: Vec, -} - -impl<'py> TreeVisitor, usize, PyErr> for DependencyTree<'py> { - fn key(&mut self, node: &Bound<'py, PyAny>) -> PyResult { - if let Some(identity) = self.identities.get_item(node)? { - return identity.extract(); - } - let identity: usize = self.identities.len(); - self.identities.set_item(node, identity)?; - Ok(identity) - } - - fn line(&mut self, line: TreeLine<'_, Bound<'py, PyAny>>) -> PyResult<()> { - let suffix: String = if line.repeated { - text(&self.already_shown.call0()?)? - } else { - String::new() - }; - let branch: String = text( - &self - .branch_style - .call1((format!("{}{}", line.prefix, line.branch),))?, - )?; - let node: String = text(&self.format_node.call1((line.node,))?)?; - self.lines.push(format!("{branch}{node}{suffix}")); - Ok(()) - } - - fn children(&mut self, node: &Bound<'py, PyAny>) -> PyResult>> { - let children: Bound<'py, PyAny> = self - .deps - .call_method1("get", (node, PyTuple::empty(node.py())))?; - let kwargs: Bound<'py, PyDict> = PyDict::new(node.py()); - kwargs.set_item("key", &self.sort_key)?; - let sorted: Bound<'py, PyAny> = self.sorted.call((children,), Some(&kwargs))?; - Ok(sorted.downcast::()?.iter().collect()) - } -} - -/// Column-trace edges shown once per related column id, sorted by that id. -struct ColumnTraceTree<'py> { - deps: Bound<'py, PyAny>, - column_id: Bound<'py, PyAny>, - related_column: Bound<'py, PyAny>, - format_related: Bound<'py, PyAny>, - branch_style: Bound<'py, PyAny>, - already_shown: Bound<'py, PyAny>, - last_key: String, - lines: Vec, -} - -impl<'py> ColumnTraceTree<'py> { - fn related_id(&self, edge: &Bound<'py, PyAny>) -> PyResult { - text( - &self - .column_id - .call1((self.related_column.call1((edge,))?,))?, - ) - } - - /// `deps.get(column_id, ())` sorted by each edge's related column id, stably. - fn sorted_edges(&self, column_id: &str) -> PyResult>> { - let py: Python<'py> = self.deps.py(); - let edges: Bound<'py, PyAny> = self - .deps - .call_method1("get", (PyString::new(py, column_id), PyTuple::empty(py)))?; - let mut keyed: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); - for edge in edges.try_iter()? { - let edge: Bound<'py, PyAny> = edge?; - keyed.push((self.related_id(&edge)?, edge)); - } - keyed.sort_by(|left, right| left.0.cmp(&right.0)); - Ok(keyed.into_iter().map(|(_, edge)| edge).collect()) - } -} - -impl<'py> TreeVisitor, String, PyErr> for ColumnTraceTree<'py> { - fn key(&mut self, edge: &Bound<'py, PyAny>) -> PyResult { - self.last_key = self.related_id(edge)?; - Ok(self.last_key.clone()) - } - - fn line(&mut self, line: TreeLine<'_, Bound<'py, PyAny>>) -> PyResult<()> { - let suffix: String = if line.repeated { - text(&self.already_shown.call0()?)? - } else { - String::new() - }; - let branch: String = text(&self.branch_style.call1((line.branch,))?)?; - let related: String = text(&self.format_related.call1((line.node,))?)?; - self.lines - .push(format!("{}{branch}{related}{suffix}", line.prefix)); - Ok(()) - } - - fn children(&mut self, _edge: &Bound<'py, PyAny>) -> PyResult>> { - let key: String = self.last_key.clone(); - self.sorted_edges(&key) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_dependency_tree, module)?)?; - module.add_function(wrap_pyfunction!(render_column_trace_tree, module)?)?; - module.add_function(wrap_pyfunction!(render_column_trace_limit_note, module)?)?; - module.add_function(wrap_pyfunction!(serialize_column, module)?)?; - module.add_function(wrap_pyfunction!(serialize_column_edge, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index c811cfd501..04923fcfd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -6,23 +6,30 @@ pub(crate) mod authored_values; pub(crate) mod boundary; pub(crate) mod cache; pub(crate) mod cli; +pub(crate) mod command_runtime; pub(crate) mod compile_attachment; pub(crate) mod compile_command; pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compiled_project; pub(crate) mod contracts; +pub(crate) mod cost_report; +pub(crate) mod cursor; pub(crate) mod dag_artifact; +pub(crate) mod debug_command; pub(crate) mod discovery; pub(crate) mod fingerprints; +pub(crate) mod freshness; pub(crate) mod frontier; pub(crate) mod graph; pub(crate) mod hand_built; +pub(crate) mod janitor; pub(crate) mod lineage; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod model_config; pub(crate) mod plan_output; +pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; pub(crate) mod refactor; @@ -36,3 +43,4 @@ pub(crate) mod semantic_shapes; pub(crate) mod sql_test_glue; pub(crate) mod sqltext; pub(crate) mod type_system; +pub(crate) mod workspace_commands; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs new file mode 100644 index 0000000000..ddae3fc318 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs @@ -0,0 +1,133 @@ +//! Typed values read from raw authored configuration mappings, rejecting the wrong types. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyBool, PyInt, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const DATETIME_MODULE: &str = "datetime"; + +/// `values.get(key)`, or `None` when the value is absent or `None`. +fn authored<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { + let value: Bound<'py, PyAny> = values.call_method1("get", (key,))?; + Ok(Some(value).filter(|value| !value.is_none())) +} + +/// `ConfigValueTypeError(key=key, expected=expected, actual_type=type(value))`. +fn wrong_type(key: &str, expected: &str, value: &Bound<'_, PyAny>) -> PyErr { + let py = value.py(); + let raised: PyResult> = + imported(py, EXCEPTIONS_MODULE, "ConfigValueTypeError").and_then(|class| { + call_with( + &class, + &[ + ("key", PyString::new(py, key).into_any()), + ("expected", PyString::new(py, expected).into_any()), + ("actual_type", value.get_type().into_any()), + ], + ) + }); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// An authored string, preserving a present empty string. +#[pyfunction(name = "get_config_str")] +#[pyo3(signature = (*, values, key))] +fn config_str<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { + let Some(value) = authored(values, key)? else { + return Ok(None); + }; + if !value.is_instance_of::() { + return Err(wrong_type(key, "a string", &value)); + } + Ok(Some(value)) +} + +/// An authored boolean, or `None` when the key is absent or `None`. +#[pyfunction(name = "get_config_bool")] +#[pyo3(signature = (*, values, key))] +fn config_bool<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { + let Some(value) = authored(values, key)? else { + return Ok(None); + }; + if !value.is_instance_of::() { + return Err(wrong_type(key, "a boolean", &value)); + } + Ok(Some(value)) +} + +/// An authored sequence of strings as a tuple, or `None` when absent or `None`. +#[pyfunction(name = "get_config_string_tuple")] +#[pyo3(signature = (*, values, key))] +fn config_string_tuple<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult>> { + let Some(value) = authored(values, key)? else { + return Ok(None); + }; + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Err(wrong_type(key, "a list or tuple of strings", &value)); + } + let items: Vec> = value.try_iter()?.collect::>()?; + if !items.iter().all(|item| item.is_instance_of::()) { + return Err(wrong_type(key, "a list or tuple of strings", &value)); + } + PyTuple::new(values.py(), items).map(Some) +} + +/// An authored integer, excluding booleans, or `None` when absent or `None`. +#[pyfunction(name = "get_config_int")] +#[pyo3(signature = (*, values, key))] +fn config_int<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { + let Some(value) = authored(values, key)? else { + return Ok(None); + }; + if value.is_instance_of::() || !value.is_instance_of::() { + return Err(wrong_type(key, "an integer", &value)); + } + Ok(Some(value)) +} + +/// A cursor bound as text from a string, integer, date or datetime. +#[pyfunction(name = "get_config_cursor_bound")] +#[pyo3(signature = (*, values, key))] +fn config_cursor_bound<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult>> { + let Some(value) = authored(values, key)? else { + return Ok(None); + }; + let py = values.py(); + let date: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "date")?; + let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + let dated: bool = value.is_instance(&date)? || value.is_instance(&datetime)?; + if value.is_instance_of::() + || !(dated || value.is_instance_of::() || value.is_instance_of::()) + { + return Err(wrong_type( + key, + "a string, integer, date, or datetime", + &value, + )); + } + if dated { + return value.call_method0("isoformat").map(Some); + } + value.str().map(|text| Some(text.into_any())) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(config_str, module)?)?; + module.add_function(wrap_pyfunction!(config_bool, module)?)?; + module.add_function(wrap_pyfunction!(config_string_tuple, module)?)?; + module.add_function(wrap_pyfunction!(config_int, module)?)?; + module.add_function(wrap_pyfunction!(config_cursor_bound, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index 17a798ff26..bec5690c43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod authored_nodes; pub(crate) mod column_migrations; pub(crate) mod config_errors; pub(crate) mod config_templates; +pub(crate) mod config_values; pub(crate) mod described_models; pub(crate) mod errors; pub(crate) mod header_columns; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index 779d8d59d8..d56dbf58c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -3,14 +3,15 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ - column_migrations, config_errors, config_templates, described_models, header_columns, - model_config_builder, model_validator, + column_migrations, config_errors, config_templates, config_values, described_models, + header_columns, model_config_builder, model_validator, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { column_migrations::register(module)?; config_errors::register(module)?; config_templates::register(module)?; + config_values::register(module)?; described_models::register(module)?; header_columns::register(module)?; model_config_builder::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index 264356778f..51239c58ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -4,8 +4,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_outputs::json_report::indented_json; +use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::plan_output::cursor_details::{ CursorDetails, cursor_details, owner, }; @@ -17,7 +18,6 @@ use crate::bindings::_helpers::plan_output::values::{ any, dict, enum_value, iso, items, list, mapped, optional_enum_value, relation_name, text, }; -const SENTINEL_TOKEN_MODULE: &str = "sqlbuild.cursor_algebra.main.sentinel_to_token"; const MICROBATCH_MODE: &str = "microbatch"; const RENAMED_REASON: &str = "renamed"; const RENAMED_DECISION: &str = "renamed"; @@ -25,10 +25,7 @@ const VIEW_MATERIALIZATION: &str = "view"; const CHANGED_IDENTITY: &str = "changed"; fn token<'py>(bound: Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = bound.py(); - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("sentinel", bound)?; - imported(py, SENTINEL_TOKEN_MODULE, "sentinel_to_token")?.call((), Some(&keywords)) + Ok(PyString::new(bound.py(), &sentinel_to_token(&bound)?).into_any()) } fn token_bounds<'py>(bounds: &Bound<'py, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs index 1feb11dc79..308ae74ccb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs @@ -1,7 +1,7 @@ //! The text plan: planner objects read by attribute into the native plan report. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::main::format_plan_completion::format_plan_completion; use sqlbuild_core::cli_text::main::format_plan_text::format_plan_text; @@ -14,6 +14,7 @@ use sqlbuild_core::cli_text::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::plan_output::cursor_details::{ CursorDetails, DEFERRED, RESOLVED, cursor_details, runtime_inputs_clause, }; @@ -32,7 +33,6 @@ const INCLUSIVE_END_MODULE: &str = "sqlbuild.compiler.planner.main.execution.inc const MATERIALIZATION_LABEL_MODULE: &str = "sqlbuild.compiler.planner.main.execution._model_materialization_label"; const PRE_BUILD_MODULE: &str = "sqlbuild.compiler.planner.main.pre_build.pre_build_work_display"; -const SENTINEL_TOKEN_MODULE: &str = "sqlbuild.cursor_algebra.main.sentinel_to_token"; const DISPLAY_MODELS_MODULE: &str = "sqlbuild.presentation.models"; const SKIP_ACTION: &str = "skip"; const QUERY_CHANGED: &str = "query_changed"; @@ -65,10 +65,7 @@ pub(crate) fn query_diff( } fn token<'py>(bound: Bound<'py, PyAny>) -> PyResult> { - call_with( - &imported(bound.py(), SENTINEL_TOKEN_MODULE, "sentinel_to_token")?, - &[("sentinel", bound)], - ) + Ok(PyString::new(bound.py(), &sentinel_to_token(&bound)?).into_any()) } /// `start → inclusive end` of resolved bounds, in the model's cursor display. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs index 0280c81335..4df340d178 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs @@ -1,20 +1,15 @@ //! Structured future-cursor and maximum-start safety evidence for plan and result JSON. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; +use pyo3::types::{PyDict, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::scalars::render_value; use crate::bindings::_helpers::plan_output::values::{dict, enum_value, items}; -const RENDER_MODULE: &str = "sqlbuild.cursor_algebra.main.render"; - /// `render(value=value)`. fn render<'py>(value: Bound<'py, PyAny>) -> PyResult> { - call_with( - &imported(value.py(), RENDER_MODULE, "render")?, - &[("value", value)], - ) + Ok(PyString::new(value.py(), &render_value(&value)?).into_any()) } /// `render(value=value)` when set, else `None`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs new file mode 100644 index 0000000000..07a4a354d9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs @@ -0,0 +1,98 @@ +//! `CliDocument`: simple static CLI output built line by line. + +use std::sync::{Mutex, MutexGuard}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliDocument as Document; + +use crate::bindings::_helpers::presentation_text::functions::style_of; + +/// Build simple static CLI output with semantic styling. +#[pyclass(module = "sqlbuild._native", name = "CliDocument", frozen)] +pub(crate) struct CliDocument { + document: Mutex, +} + +impl CliDocument { + fn lines(&self) -> PyResult> { + self.document + .lock() + .map_err(|_| PyRuntimeError::new_err("CLI document lock is poisoned")) + } +} + +#[pymethods] +impl CliDocument { + #[new] + fn new(style: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + document: Mutex::new(Document::new(style_of(style)?)), + }) + } + + #[pyo3(signature = (*, text, suffix = None))] + fn header(&self, text: &str, suffix: Option<&str>) -> PyResult<()> { + self.lines()?.header(text, suffix); + Ok(()) + } + + fn blank(&self) -> PyResult<()> { + self.lines()?.lines.push(String::new()); + Ok(()) + } + + #[pyo3(signature = (*, label, value, value_padding = " "))] + fn field(&self, label: &str, value: &str, value_padding: &str) -> PyResult<()> { + self.lines()?.field(label, (value, value_padding)); + Ok(()) + } + + #[pyo3(signature = (*, rows, label_width = None))] + fn fields(&self, rows: Vec<(String, String)>, label_width: Option) -> PyResult<()> { + self.lines()?.fields(&rows, label_width); + Ok(()) + } + + fn section(&self, text: &str) -> PyResult<()> { + self.lines()?.section(text); + Ok(()) + } + + fn title_section(&self, text: &str) -> PyResult<()> { + self.lines()?.title_section(text); + Ok(()) + } + + fn line(&self, text: &str) -> PyResult<()> { + self.lines()?.lines.push(text.to_owned()); + Ok(()) + } + + fn items(&self, values: Vec) -> PyResult<()> { + self.lines()?.items(&values); + Ok(()) + } + + #[pyo3(signature = (*, prefix, command, style_command = true))] + fn command_line(&self, prefix: &str, command: &str, style_command: bool) -> PyResult<()> { + self.lines()?.command_line(prefix, (command, style_command)); + Ok(()) + } + + #[pyo3(signature = (*, commands, numbered = false, style_command = true))] + fn commands(&self, commands: Vec, numbered: bool, style_command: bool) -> PyResult<()> { + self.lines()?.commands(&commands, (numbered, style_command)); + Ok(()) + } + + #[pyo3(signature = (*, trailing_newline = true))] + fn render(&self, trailing_newline: bool) -> PyResult { + Ok(self.lines()?.render(trailing_newline)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs new file mode 100644 index 0000000000..983c9bf881 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs @@ -0,0 +1,203 @@ +//! Shared CLI text helpers as `_native` functions; styles are read from a `CliStyle`'s `use_color`. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyList, PyListMethods, PySlice}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::cli_text::main::format_aligned_name_value::format_aligned_name_value as aligned; +use sqlbuild_core::cli_text::main::format_completion_line::format_completion_line as completion; +use sqlbuild_core::cli_text::main::format_count_noun::format_count_noun as count_noun; +use sqlbuild_core::cli_text::main::format_overflow_line::format_overflow_line; +use sqlbuild_core::cli_text::main::format_status_cell::format_status_cell as status_cell; +use sqlbuild_core::cli_text::main::format_summary_footer::format_summary_footer as footer; +use sqlbuild_core::cli_text::main::resolve_name_column_width::resolve_name_column_width as column_width; +use sqlbuild_core::cli_text::main::split_count_header::split_count_header; +use sqlbuild_core::cli_text::models::{AlignedRow, CliStyle, CompletionState, SummaryCount}; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; + +const ALIGNED_OPTIONS: (&[&str], &[&str]) = ( + &["plain_name", "styled_name", "value", "name_column_width"], + &["prefix"], +); +const OVERFLOW_OPTIONS: (&[&str], &[&str]) = ( + &["lines", "total_count", "visible_count", "indent", "options"], + &[], +); +const DEFAULT_PREFIX: &str = " "; +const FAIL_STATE: &str = "fail"; +const WARN_STATE: &str = "warn"; + +/// The native style of a Python `CliStyle`. +pub(crate) fn style_of(style: &Bound<'_, PyAny>) -> PyResult { + Ok(CliStyle { + use_color: style.getattr("use_color")?.is_truthy()?, + }) +} + +/// Format summary counts with semantic colors. +#[pyfunction] +#[pyo3(signature = (*, counts, use_color, elapsed = None))] +fn format_summary_footer( + counts: Vec<(String, i64)>, + use_color: bool, + elapsed: Option<&str>, +) -> String { + let counts: Vec = counts + .into_iter() + .map(|(label, value)| SummaryCount { label, value }) + .collect(); + footer(CliStyle { use_color }, &counts, elapsed) +} + +/// Render a count with the singular noun for one and the plural noun otherwise. +#[pyfunction] +#[pyo3(signature = (*, count, singular, plural = None))] +fn format_count_noun(count: i64, singular: &str, plural: Option<&str>) -> String { + count_noun(count, singular, plural) +} + +/// Render a single-line completion summary: state glyph, label, trailing summary. +#[pyfunction] +#[pyo3(signature = (*, style, state, label, summary = None))] +fn format_completion_line( + style: &Bound<'_, PyAny>, + state: &str, + label: &str, + summary: Option<&str>, +) -> PyResult { + let state: CompletionState = match state { + FAIL_STATE => CompletionState::Fail, + WARN_STATE => CompletionState::Warn, + _ => CompletionState::Ok, + }; + Ok(completion(style_of(style)?, state, label, summary)) +} + +/// Render a fixed-width status cell padded on the plain text, not the ANSI text. +#[pyfunction] +#[pyo3(signature = (*, style, status, width = 6))] +fn format_status_cell(style: &Bound<'_, PyAny>, status: &str, width: usize) -> PyResult { + Ok(status_cell(style_of(style)?, status, width)) +} + +/// Format one aligned name/value row using a plain-text width basis. +#[pyfunction] +#[pyo3(signature = (**options))] +fn format_aligned_name_value( + py: Python<'_>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "format_aligned_name_value", options, ALIGNED_OPTIONS)?; + let prefix: String = if arguments.passed("prefix")? { + arguments.value("prefix")?.extract()? + } else { + DEFAULT_PREFIX.to_owned() + }; + let width: i64 = arguments.value("name_column_width")?.extract()?; + Ok(aligned(&AlignedRow { + plain_name: arguments.value("plain_name")?.extract()?, + styled_name: arguments.value("styled_name")?.extract()?, + value: arguments.value("value")?.extract()?, + name_column_width: usize::try_from(width).unwrap_or(0), + prefix, + })) +} + +/// Resolve a left-column width from the longest visible name. +#[pyfunction] +#[pyo3(signature = (*, names, min_width = 20))] +fn resolve_name_column_width(names: Vec, min_width: usize) -> usize { + column_width(&names, min_width) +} + +/// Append a standard overflow hint when a section is truncated, and return the lines. +#[pyfunction] +#[pyo3(signature = (**options))] +fn append_overflow_line<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "append_overflow_line", options, OVERFLOW_OPTIONS)?; + let lines: Bound<'py, PyList> = arguments.value("lines")?.downcast_into()?; + let total: i64 = arguments.value("total_count")?.extract()?; + let visible: i64 = arguments.value("visible_count")?.extract()?; + let indent: String = arguments.value("indent")?.extract()?; + let flag: String = arguments + .value("options")? + .getattr("overflow_flag")? + .extract()?; + if let Some(line) = format_overflow_line(total - visible, &indent, &flag) { + lines.append(line)?; + } + Ok(lines) +} + +/// Return entries visible under the current display cap. +#[pyfunction] +#[pyo3(signature = (*, entries, options))] +fn visible_entries<'py>( + entries: Bound<'py, PyAny>, + options: &Bound<'py, PyAny>, +) -> PyResult> { + let limit: Bound<'py, PyAny> = options.getattr("max_entries_per_section")?; + if limit.is_none() { + return Ok(entries); + } + let slice: Bound<'py, PySlice> = PySlice::new(entries.py(), 0, limit.extract()?, 1); + entries.get_item(slice) +} + +/// A header style that renders `Title (count)` as a styled title and a dim count. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CountHeaderStyle { + style: CliStyle, + title_style: Option>, +} + +#[pymethods] +impl CountHeaderStyle { + fn __call__(&self, py: Python<'_>, text: &str) -> PyResult { + let title = |title: &str| -> PyResult { + match &self.title_style { + Some(styled) => styled.bind(py).call1((title,))?.extract(), + None => Ok(self.style.section(title)), + } + }; + let Some((heading, count)) = split_count_header(text) else { + return title(text); + }; + Ok(format!( + "{} {}", + title(heading)?, + self.style.muted(&format!("({count}")) + )) + } +} + +/// Return a header style that renders `Title (count)` as a bold title and dim count. +#[pyfunction] +#[pyo3(signature = (*, style, title_style = None))] +fn count_header_style( + style: &Bound<'_, PyAny>, + title_style: Option>, +) -> PyResult { + Ok(CountHeaderStyle { + style: style_of(style)?, + title_style, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(format_summary_footer, module)?)?; + module.add_function(wrap_pyfunction!(format_count_noun, module)?)?; + module.add_function(wrap_pyfunction!(format_completion_line, module)?)?; + module.add_function(wrap_pyfunction!(format_status_cell, module)?)?; + module.add_function(wrap_pyfunction!(format_aligned_name_value, module)?)?; + module.add_function(wrap_pyfunction!(resolve_name_column_width, module)?)?; + module.add_function(wrap_pyfunction!(append_overflow_line, module)?)?; + module.add_function(wrap_pyfunction!(visible_entries, module)?)?; + module.add_function(wrap_pyfunction!(count_header_style, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs new file mode 100644 index 0000000000..519415c3d5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod document; +pub(crate) mod functions; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs new file mode 100644 index 0000000000..999bafd567 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs @@ -0,0 +1,10 @@ +//! Register the shared CLI text helpers. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::presentation_text::{document, functions}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + document::register(module)?; + functions::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs index 4b42448cb5..8e8a05ddf7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod contract_compare; pub(crate) mod contract_edits; pub(crate) mod python_strings; +pub(crate) mod refactor_command; +pub(crate) mod refactor_facts; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs new file mode 100644 index 0000000000..8cfc8ae1cc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -0,0 +1,451 @@ +//! `sqb rename` and `sqb mv`: plan from compiler facts, verify in a scratch copy, then write. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::lineage_command::models::LineageKey; +use sqlbuild_core::cli_text::main::format_count_noun::format_count_noun; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_refactor::refactor_command::main::layer_move_suggestion::layer_move_suggestion; +use sqlbuild_refactor::refactor_command::main::refactor_request_for_target::refactor_request_for_target; +use sqlbuild_refactor::refactor_command::main::render_refactor_json::render_refactor_json; +use sqlbuild_refactor::refactor_command::main::render_refactor_text::render_refactor_text; +use sqlbuild_refactor::refactor_command::models::{ + LayerMove, RefactorDiagnostic, RefactorStatus, RefactorTargetRequest, +}; +use sqlbuild_refactor::refactoring::errors::{RefactorError, RefactorErrorKind}; +use sqlbuild_refactor::refactoring::main::commit_refactor_plan::commit_refactor_plan; +use sqlbuild_refactor::refactoring::main::plan_refactor::plan_refactor; +use sqlbuild_refactor::refactoring::main::stage_refactor_plan::stage_refactor_plan; +use sqlbuild_refactor::refactoring::main::with_model_migration::with_model_migration; +use sqlbuild_refactor::refactoring::models::{ + DeclarationPlacement, ModelFacts, RefactorFacts, RefactorOperation, RefactorPlan, + RefactorRequest, +}; +use sqlbuild_refactor::refactoring::types::Originals; + +use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_guard}; +use crate::bindings::_helpers::lineage::lineage_index::lineage_key; +use crate::bindings::_helpers::refactor::refactor_facts::{ + declaration_placement, model_facts, refactor_facts, +}; + +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const REFACTOR_ERRORS_MODULE: &str = "sqlbuild.compiler.refactoring.exceptions"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const STATUS_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; +const COMPILE_MODULE: &str = "sqlbuild.cli.commands._helpers.refactor.compile_facts"; +const STAGING_PREFIX: &str = "sqb-refactor-"; +const MOVE_COMMAND: &str = "mv"; +const EDIT_ERROR_CODE: &str = "C957"; +const WRITE_ERROR_CODE: &str = "C958"; + +fn imported<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { + py.import(module)?.getattr(name) +} + +/// `CliUserError(message, code=..., help=...)`. +fn user_error(py: Python<'_>, (message, code, help): (String, &str, Option)) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", code)?; + options.set_item("help", help)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// The Python exception a native refactoring failure becomes. +fn refactor_error(py: Python<'_>, error: RefactorError) -> PyErr { + let class = |name: &str| -> PyErr { + match imported(py, REFACTOR_ERRORS_MODULE, name) + .and_then(|class| class.call1((error.message.clone(),))) + { + Ok(raised) => PyErr::from_value(raised), + Err(failure) => failure, + } + }; + match error.kind { + RefactorErrorKind::Input => user_error(py, (error.message, &error.code, error.help)), + RefactorErrorKind::Edit => user_error(py, (error.message, EDIT_ERROR_CODE, None)), + RefactorErrorKind::Write => user_error(py, (error.message, WRITE_ERROR_CODE, error.help)), + RefactorErrorKind::Value => class("RefactorValueError"), + RefactorErrorKind::Io => class("RefactorIOError"), + RefactorErrorKind::Internal => NativeCompilerError::new_err(error.message), + } +} + +/// One offline compile: the Python `RefactorCompile`. +struct Compiled<'py> { + project: Bound<'py, PyAny>, + errors: Vec>, +} + +fn compile<'py>(py: Python<'py>, project_dir: &Path, no_cache: bool) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("project_dir", project_dir)?; + options.set_item("no_cache", no_cache)?; + let compiled: Bound<'py, PyAny> = + imported(py, COMPILE_MODULE, "compile_for_refactor")?.call((), Some(&options))?; + Ok(Compiled { + project: compiled.getattr("project")?, + errors: compiled + .getattr("errors")? + .try_iter()? + .collect::>()?, + }) +} + +fn optional_posix(path: &Bound<'_, PyAny>) -> PyResult> { + if path.is_none() { + return Ok(None); + } + Ok(Some(path.call_method0("as_posix")?.extract()?)) +} + +/// The edited project's errors, paths inside the scratch copy made relative to it. +fn relative_diagnostics( + errors: &[Bound<'_, PyAny>], + staging_dir: &Path, +) -> PyResult> { + let mut diagnostics: Vec = Vec::new(); + for error in errors { + let path: Option = optional_posix(&error.getattr("path")?)?.map(|path| { + let candidate: &Path = Path::new(&path); + match candidate.strip_prefix(staging_dir) { + Ok(relative) if candidate.is_absolute() => relative.to_string_lossy().into_owned(), + _ => path, + } + }); + diagnostics.push(RefactorDiagnostic { + code: error.getattr("code")?.extract()?, + message: error.getattr("message")?.extract()?, + path, + line: error.getattr("line")?.extract()?, + }); + } + Ok(diagnostics) +} + +/// Print each compile error to stderr as `error[code] path: message`. +fn write_errors(py: Python<'_>, errors: &[Bound<'_, PyAny>]) -> PyResult<()> { + let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; + for error in errors { + let location: String = optional_posix(&error.getattr("path")?)? + .map_or_else(String::new, |path| format!("{path}: ")); + let code: String = error.getattr("code")?.str()?.extract()?; + let message: String = error.getattr("message")?.str()?.extract()?; + stderr.call_method1("write", (format!("error[{code}] {location}{message}\n"),))?; + } + Ok(()) +} + +fn files(plan: &RefactorPlan) -> String { + format_count_noun( + i64::try_from(plan.changes.len()).unwrap_or(i64::MAX), + "file", + None, + ) +} + +/// The Python transient status reporter on the command's progress stream. +struct Status<'py> { + reporter: Bound<'py, PyAny>, +} + +impl Status<'_> { + fn start(&self, message: &str) -> PyResult<()> { + self.reporter.call_method1("start", (message,))?; + Ok(()) + } + + fn complete(&self, message: &str) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(self.reporter.py()); + options.set_item("message", message)?; + self.reporter.call_method("complete", (), Some(&options))?; + Ok(()) + } + + fn error(&self, message: &str) -> PyResult<()> { + self.reporter.call_method1("error", (message,))?; + Ok(()) + } + + fn close(&self) -> PyResult<()> { + self.reporter.call_method0("close")?; + Ok(()) + } +} + +/// The command's options and output settings. +struct Command<'py> { + request: Bound<'py, PyAny>, + project_dir: PathBuf, + use_color: bool, + json_output: bool, +} + +/// Where a refactoring stopped: its plan, terminal status and compile errors. +struct Outcome { + plan: RefactorPlan, + status: RefactorStatus, + diagnostics: Vec, +} + +/// A plan verified in the scratch copy, with its originals and the edited project's errors. +struct Verified { + plan: RefactorPlan, + originals: Originals, + diagnostics: Vec, +} + +fn target_lookup(all_keys: &Bound<'_, PyAny>) -> PyResult> { + let mut names: HashMap = HashMap::new(); + for item in all_keys.call_method0("items")?.try_iter()? { + let (name, key): (String, Bound<'_, PyAny>) = item?.extract()?; + names.insert(name, lineage_key(&key)?); + } + Ok(names) +} + +fn target_request(request: &Bound<'_, PyAny>) -> PyResult { + Ok(RefactorTargetRequest { + is_move: request.getattr("command")?.str()?.extract::()? == MOVE_COMMAND, + target: request.getattr("target")?.extract()?, + new_name: request.getattr("new_name")?.extract()?, + destination: request.getattr("destination")?.extract()?, + cascade: request.getattr("cascade")?.extract()?, + }) +} + +fn plan( + py: Python<'_>, + project: &Bound<'_, PyAny>, + request: &RefactorRequest, +) -> PyResult { + let facts: RefactorFacts = refactor_facts(project, request)?; + let host_error: RefCell> = RefCell::new(None); + let host = |model: &str, destination: &str| -> Result { + declaration_placement(project, (model, destination)).map_err(|error| { + host_error.replace(Some(error)); + RefactorError::internal("the declaration-move host failed") + }) + }; + let planned: Result = plan_refactor(&facts, request, &host); + if let Some(error) = host_error.into_inner() { + return Err(error); + } + planned.map_err(|error| refactor_error(py, error)) +} + +fn stage( + py: Python<'_>, + (project_dir, staging_dir): (&Path, &Path), + plan: &RefactorPlan, + copy_inputs: bool, +) -> PyResult { + stage_refactor_plan(project_dir, staging_dir, plan, copy_inputs) + .map_err(|error| refactor_error(py, error)) +} + +/// Verify the plan in a scratch copy, adding model migrations when the relation moves. +fn verify( + py: Python<'_>, + command: &Command<'_>, + (before, mut plan): (&Bound<'_, PyAny>, RefactorPlan), + (staging_dir, status): (&Path, &Status<'_>), +) -> PyResult> { + let project_dir: &Path = &command.project_dir; + status.start("Verifying the edited project compiles...")?; + let mut originals: Originals = stage(py, (project_dir, staging_dir), &plan, true)?; + let mut after: Compiled<'_> = compile(py, staging_dir, true)?; + if plan.request.operation != RefactorOperation::RenameColumn { + let before_facts: Vec = model_facts(before)?.unwrap_or_default(); + let after_facts: Option> = model_facts(&after.project)?; + let migrated: RefactorPlan = + with_model_migration(&plan, &before_facts, after_facts.as_deref(), &originals); + if !migrated.blocking.is_empty() { + status.error("Refused: the renamed model's history cannot be kept.")?; + return Ok(Err(Outcome { + plan: migrated, + status: RefactorStatus::Refused, + diagnostics: Vec::new(), + })); + } + if migrated.migrations != plan.migrations { + plan = migrated; + originals = stage(py, (project_dir, staging_dir), &plan, false)?; + after = compile(py, staging_dir, true)?; + } + } + let diagnostics: Vec = relative_diagnostics(&after.errors, staging_dir)?; + Ok(Ok(Verified { + plan, + originals, + diagnostics, + })) +} + +fn run(py: Python<'_>, command: &Command<'_>, status: &Status<'_>) -> PyResult> { + let started: Instant = Instant::now(); + status.start("Compiling project...")?; + let before: Compiled<'_> = compile(py, &command.project_dir, false)?; + if before.project.is_none() || !before.errors.is_empty() { + status.error("Project does not compile; fix it before renaming (sqb compile)")?; + write_errors(py, &before.errors)?; + return Ok(None); + } + status.complete(&format!( + "Compiled project. ({:.2}s)", + started.elapsed().as_secs_f64() + ))?; + let names: HashMap = + target_lookup(&before.project.getattr("graph")?.getattr("all_keys")?)?; + let lookup = |name: &str| names.get(name).cloned(); + let request: RefactorRequest = + refactor_request_for_target(&target_request(&command.request)?, &lookup) + .map_err(|error| refactor_error(py, error))?; + status.start("Planning edits...")?; + let planned: RefactorPlan = plan(py, &before.project, &request)?; + status.complete(&format!("Planned edits to {}.", files(&planned)))?; + if !planned.blocking.is_empty() || !planned.manual.is_empty() { + status.error("Refused: some references cannot be rewritten safely.")?; + return Ok(Some(Outcome { + plan: planned, + status: RefactorStatus::Refused, + diagnostics: Vec::new(), + })); + } + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("prefix", STAGING_PREFIX)?; + let staging: Bound<'_, PyAny> = + imported(py, "tempfile", "TemporaryDirectory")?.call((), Some(&options))?; + let staging_dir: PathBuf = staging.getattr("name")?.extract()?; + let verified = verify( + py, + command, + (&before.project, planned), + (&staging_dir, status), + ); + staging.call_method0("cleanup")?; + let Verified { + plan, + originals, + diagnostics, + } = match verified? { + Ok(verified) => verified, + Err(outcome) => return Ok(Some(outcome)), + }; + if !diagnostics.is_empty() { + status.error("Verification failed: the edited project does not compile.")?; + return Ok(Some(Outcome { + plan, + status: RefactorStatus::CompileFailed, + diagnostics, + })); + } + status.complete("Verified: the edited project compiles.")?; + if command.request.getattr("dry_run")?.extract::()? { + return Ok(Some(Outcome { + plan, + status: RefactorStatus::DryRun, + diagnostics: Vec::new(), + })); + } + status.start(&format!("Writing {}...", files(&plan)))?; + commit_refactor_plan(&command.project_dir, &originals, &plan) + .map_err(|error| refactor_error(py, error))?; + status.complete(&format!("Wrote {}.", files(&plan)))?; + Ok(Some(Outcome { + plan, + status: RefactorStatus::Applied, + diagnostics: Vec::new(), + })) +} + +fn finish(py: Python<'_>, command: &Command<'_>, outcome: &Outcome) -> PyResult { + let output: String = if command.json_output { + render_refactor_json(&outcome.plan, outcome.status, &outcome.diagnostics) + .map_err(|error| PyValueError::new_err(format!("{error:?}")))? + } else { + let project_dir: Bound<'_, PyAny> = command.request.getattr("project_dir")?; + let project_dir: Option = if project_dir.is_none() { + None + } else { + Some(project_dir.str()?.extract()?) + }; + let layer_move: Option = layer_move_suggestion( + &outcome.plan, + &outcome.diagnostics, + (project_dir.as_deref(), cfg!(windows)), + ); + render_refactor_text( + &outcome.plan, + (outcome.status, &outcome.diagnostics), + layer_move.as_ref(), + CliStyle { + use_color: command.use_color, + }, + ) + }; + py.import("sys")? + .getattr("stdout")? + .call_method1("write", (output,))?; + Ok(i32::from(!matches!( + outcome.status, + RefactorStatus::Applied | RefactorStatus::DryRun + ))) +} + +/// Plan, verify in a scratch copy, and write one rename or move. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_refactor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let project_dir: Bound<'_, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, + }; + let json_output: bool = request.getattr("json_output")?.extract()?; + let use_color: bool = !request.getattr("no_color")?.extract::()? + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .extract::()?; + let command: Command<'_> = Command { + request: request.clone(), + project_dir: project_dir.call_method0("resolve")?.extract()?, + use_color, + json_output, + }; + let stream: Bound<'_, PyAny> = + py.import("sys")? + .getattr(if json_output { "stderr" } else { "stdout" })?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", stream)?; + options.set_item("use_color", use_color)?; + let status: Status<'_> = Status { + reporter: imported(py, STATUS_MODULE, "TransientStatusReporter")? + .call((), Some(&options))?, + }; + let result: PyResult> = compiler_guard(|| run(py, &command, &status)); + status.close()?; + match result? { + None => Ok(1), + Some(outcome) => finish(py, &command, &outcome), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_refactor_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs new file mode 100644 index 0000000000..18b161d1d6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs @@ -0,0 +1,390 @@ +//! Refactoring facts read from a compiled project, and the shared scope placement host. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_refactor::refactoring::models::{ + AuthoredFile, DeclarationPlacement, DependencyFacts, DestinationFacts, DiscoveredFile, + ExpansionFacts, ExpansionSpan, LocationFacts, ManualLocation, ModelFacts, PlacedDeclaration, + RefactorFacts, RefactorOperation, RefactorRequest, RelocatedDeclaration, ResourceColumnsFacts, + SchemaColumnFacts, SqlFileRole, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const CONFIG_MODULE: &str = "sqlbuild._native"; +const SCOPE_MODELS: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_TYPES: &str = "sqlbuild.compiler.scopes.types"; +const GENERIC_DIALECT: &str = "generic"; +const MATERIALIZED_KEY: &str = "materialized"; +const MIGRATE_FROM_KEY: &str = "migrate_from"; +const MODEL_KIND_PREFIX: &str = "model:"; + +fn posix(path: &Bound<'_, PyAny>) -> PyResult { + path.call_method0("as_posix")?.extract() +} + +fn native<'py>(py: Python<'py>, name: &str) -> PyResult> { + py.import(NATIVE_MODULE)?.getattr(name) +} + +fn discovered_files(files: &Bound<'_, PyAny>) -> PyResult> { + let mut discovered: Vec = Vec::new(); + for item in files.try_iter()? { + let item: Bound<'_, PyAny> = item?; + discovered.push(DiscoveredFile { + path: posix(&item.getattr("relative_path")?)?, + contents: item.getattr("contents")?.extract()?, + }); + } + Ok(discovered) +} + +fn column_names(columns: &Bound<'_, PyAny>) -> PyResult> { + columns + .try_iter()? + .map(|column| column?.getattr("name")?.extract()) + .collect() +} + +fn resource_columns( + resources: &Bound<'_, PyAny>, + entry: &str, +) -> PyResult> { + let mut facts: Vec = Vec::new(); + for resource in resources.try_iter()? { + let resource: Bound<'_, PyAny> = resource?; + facts.push(ResourceColumnsFacts { + name: resource.getattr("name")?.extract()?, + columns: column_names(&resource.getattr(entry)?.getattr("columns")?)?, + }); + } + Ok(facts) +} + +fn spans(spans: &Bound<'_, PyAny>) -> PyResult> { + let mut read: Vec = Vec::new(); + for span in spans.try_iter()? { + let span: Bound<'_, PyAny> = span?; + read.push(ExpansionSpan { + source_start: span.getattr("source_start")?.extract()?, + source_end: span.getattr("source_end")?.extract()?, + output_start: span.getattr("output_start")?.extract()?, + output_end: span.getattr("output_end")?.extract()?, + }); + } + Ok(read) +} + +fn expansion_facts(expansion: &Bound<'_, PyAny>) -> PyResult { + Ok(ExpansionFacts { + expanded_sql: expansion.getattr("expanded_sql")?.extract()?, + passes: expansion + .getattr("passes")? + .try_iter()? + .map(|pass| spans(&pass?)) + .collect::>()?, + }) +} + +fn schema_columns(entry: &Bound<'_, PyAny>) -> PyResult>> { + if entry.is_none() { + return Ok(None); + } + let mut columns: Vec = Vec::new(); + for column in entry.getattr("columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let location: Bound<'_, PyAny> = column.getattr("location")?; + columns.push(SchemaColumnFacts { + name: column.getattr("name")?.extract()?, + migrate_from: !column.getattr("migrate_from")?.is_none(), + location: if location.is_none() { + None + } else { + Some(LocationFacts { + path: posix(&location.getattr("path")?)?, + line: location.getattr("line")?.extract()?, + column: location.getattr("column")?.extract()?, + }) + }, + }); + } + Ok(Some(columns)) +} + +fn model(model: &Bound<'_, PyAny>, expansion: Option<&Bound<'_, PyAny>>) -> PyResult { + let py: Python<'_> = model.py(); + let values: Bound<'_, PyDict> = PyDict::new(py); + values.call_method1("update", (model.getattr("config")?.getattr("values")?,))?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("values", &values)?; + options.set_item("key", MATERIALIZED_KEY)?; + let materialized: Option = py + .import(CONFIG_MODULE)? + .getattr("get_config_str")? + .call((), Some(&options))? + .extract()?; + let migrate_from: Option> = values.get_item(MIGRATE_FROM_KEY)?; + let mut deps: Vec = Vec::new(); + for dep in model.getattr("deps")?.try_iter()? { + let dep: Bound<'_, PyAny> = dep?; + deps.push(DependencyFacts { + resource_type: dep.getattr("resource_type")?.str()?.extract()?, + name: dep.getattr("name")?.extract()?, + }); + } + let destination: Bound<'_, PyAny> = model.getattr("destination")?; + let inferred: Bound<'_, PyAny> = model.getattr("inferred_columns")?; + Ok(ModelFacts { + name: model.getattr("name")?.extract()?, + path: posix(&model.getattr("relative_path")?)?, + deps, + query_sql: model.getattr("query_sql")?.extract()?, + authored_query_sql: model.getattr("authored_query_sql")?.extract()?, + materialized, + declares_migrate_from: migrate_from.is_some(), + migrate_from_set: migrate_from.is_some_and(|value| !value.is_none()), + destination: DestinationFacts { + database: destination.getattr("database")?.extract()?, + schema: destination.getattr("schema")?.extract()?, + name: destination.getattr("name")?.extract()?, + }, + inferred_columns: if inferred.is_none() { + Vec::new() + } else { + column_names(&inferred)? + }, + schema_columns: schema_columns(&model.getattr("schema_entry")?)?, + expansion: expansion.map(expansion_facts).transpose()?, + }) +} + +/// Every compiled model's refactoring facts, or none without a project. +pub(crate) fn model_facts( + refactor_project: &Bound<'_, PyAny>, +) -> PyResult>> { + if refactor_project.is_none() { + return Ok(None); + } + let project: Bound<'_, PyAny> = refactor_project.getattr("graph")?.getattr("project")?; + let project_dir: Bound<'_, PyAny> = refactor_project.getattr("project_dir")?; + let mut expansions: HashMap> = HashMap::new(); + let items: Vec> = project + .getattr("sql_expansions")? + .call_method0("items")? + .try_iter()? + .collect::>()?; + for item in items.iter().rev() { + let (path, expansion): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item.extract()?; + expansions.insert(path.call_method0("resolve")?.str()?.extract()?, expansion); + } + let mut facts: Vec = Vec::new(); + for compiled in project.getattr("models")?.try_iter()? { + let compiled: Bound<'_, PyAny> = compiled?; + let resolved: String = project_dir + .div(compiled.getattr("relative_path")?)? + .call_method0("resolve")? + .str()? + .extract()?; + facts.push(model(&compiled, expansions.get(&resolved))?); + } + Ok(Some(facts)) +} + +fn block_bodies(blocks: &Bound<'_, PyAny>) -> PyResult> { + blocks + .try_iter()? + .map(|block| block?.getattr("sql_body")?.extract()) + .collect() +} + +fn authored_files(discovered: &Bound<'_, PyAny>) -> PyResult> { + let groups: [(SqlFileRole, &str, Option<&str>, bool); 6] = [ + (SqlFileRole::Test, "test_files", None, true), + ( + SqlFileRole::Scenario, + "scenario_files", + Some("sql_body"), + false, + ), + (SqlFileRole::Audit, "audit_files", None, true), + (SqlFileRole::Hook, "sql_hook_files", Some("sql_body"), false), + ( + SqlFileRole::Function, + "sql_function_files", + Some("body_sql"), + false, + ), + (SqlFileRole::Schema, "model_schema_files", None, false), + ]; + let mut files: Vec = Vec::new(); + for (role, attribute, body, blocks) in groups { + for item in discovered.getattr(attribute)?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + let texts: Vec = match (body, blocks) { + (Some(body), _) => vec![item.getattr(body)?.extract()?], + (None, true) => block_bodies(&item.getattr("blocks")?)?, + (None, false) => Vec::new(), + }; + files.push(AuthoredFile { + role, + path: posix(&item.getattr("relative_path")?)?, + contents: item.getattr("contents")?.extract()?, + texts, + }); + } + } + Ok(files) +} + +fn python_locations( + project: &Bound<'_, PyAny>, + request: &RefactorRequest, +) -> PyResult> { + let py: Python<'_> = project.py(); + let old: &str = &request.model_name; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("project_dir", project.getattr("project_dir")?)?; + options.set_item("discovered", project.getattr("discovered")?)?; + if request.operation == RefactorOperation::RenameColumn { + let column: &str = request.column_name.as_deref().unwrap_or_default(); + options.set_item("names", PyTuple::new(py, [column])?)?; + options.set_item( + "reason", + format!("Python SQL may read column {column} of {old}; check it by hand"), + )?; + options.set_item("context", old)?; + } else { + options.set_item("names", PyTuple::new(py, [old])?)?; + options.set_item( + "reason", + format!("Python code names model {old}; update it by hand"), + )?; + } + let mut locations: Vec = Vec::new(); + for location in native(py, "find_python_string_locations")? + .call((), Some(&options))? + .try_iter()? + { + let location: Bound<'_, PyAny> = location?; + locations.push(ManualLocation { + path: location.getattr("path")?.extract()?, + line: location.getattr("line")?.extract()?, + column: location.getattr("column")?.extract()?, + reason: location.getattr("reason")?.extract()?, + }); + } + Ok(locations) +} + +/// The facts one refactoring plan reads. +pub(crate) fn refactor_facts( + project: &Bound<'_, PyAny>, + request: &RefactorRequest, +) -> PyResult { + let compiled: Bound<'_, PyAny> = project.getattr("graph")?.getattr("project")?; + let discovered: Bound<'_, PyAny> = project.getattr("discovered")?; + let dialect: Option = compiled.getattr("sql_analysis_dialect")?.extract()?; + let mut yaml_files: Vec = + discovered_files(&discovered.getattr("source_files")?)?; + yaml_files.extend(discovered_files(&discovered.getattr("schema_files")?)?); + Ok(RefactorFacts { + project_dir: project.getattr("project_dir")?.str()?.extract()?, + dialect: dialect + .filter(|dialect| !dialect.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()), + models: model_facts(project)?.unwrap_or_default(), + sources: resource_columns(&compiled.getattr("sources")?, "source_entry")?, + seeds: resource_columns(&compiled.getattr("seeds")?, "schema_entry")?, + model_files: discovered_files(&discovered.getattr("model_files")?)?, + authored_files: authored_files(&discovered)?, + yaml_files, + python_locations: python_locations(project, request)?, + }) +} + +fn identity_key(identity: &Bound<'_, PyAny>) -> PyResult { + identity.repr()?.extract() +} + +/// Where the shared scope placement puts declarations when `model` moves to `destination`. +pub(crate) fn declaration_placement( + project: &Bound<'_, PyAny>, + (model, destination): (&str, &str), +) -> PyResult { + let py: Python<'_> = project.py(); + let index: Bound<'_, PyAny> = project + .getattr("graph")? + .getattr("project")? + .getattr("scope_index")?; + let lookup: Bound<'_, PyAny> = native(py, "build_scope_lookup")?.call1((&index,))?; + let (preview, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = + native(py, "scope_move_preview")? + .call1((lookup, format!("{MODEL_KIND_PREFIX}{model}"), destination))? + .extract()?; + let relocated: Option> = if preview.is_none() { + None + } else { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item( + "kind", + py.import(SCOPE_TYPES)? + .getattr("ResourceKind")? + .getattr("MODEL")?, + )?; + options.set_item("name", model)?; + let resource: Bound<'_, PyAny> = py + .import(SCOPE_MODELS)? + .getattr("ResourceIdentity")? + .call((), Some(&options))?; + let records: Bound<'_, PyAny> = + native(py, "scope_relocated_declarations")?.call1((&index, resource, destination))?; + if records.is_none() { + None + } else { + let sort: Bound<'_, PyDict> = PyDict::new(py); + sort.set_item( + "key", + py.import("operator")? + .getattr("attrgetter")? + .call1(("identity",))?, + )?; + let mut relocated: Vec = Vec::new(); + for record in py + .import("builtins")? + .getattr("sorted")? + .call((records,), Some(&sort))? + .try_iter()? + { + let record: Bound<'_, PyAny> = record?; + relocated.push(RelocatedDeclaration { + key: identity_key(&record.getattr("identity")?)?, + path: record.getattr("path")?.extract()?, + }); + } + Some(relocated) + } + }; + let mut declarations: Vec = Vec::new(); + for record in index.getattr("declarations")?.try_iter()? { + let record: Bound<'_, PyAny> = record?; + let identity: Bound<'_, PyAny> = record.getattr("identity")?; + let kind: String = identity.getattr("kind")?.getattr("value")?.extract()?; + let name: String = identity.getattr("name")?.extract()?; + declarations.push(PlacedDeclaration { + key: identity_key(&identity)?, + label: format!("{kind}:{name}"), + path: record.getattr("path")?.extract()?, + line: record.getattr("line")?.extract()?, + column: record.getattr("column")?.extract()?, + }); + } + Ok(DeclarationPlacement { + relocated, + diagnostics: diagnostics + .try_iter()? + .map(|item| item?.getattr("message")?.extract()) + .collect::>()?, + declarations, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs index 9152442b55..95663e5197 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs @@ -1,150 +1,14 @@ -//! JSON bindings for native `sqb rename` and `sqb mv`; expected failures return `{"error": ...}`. +//! Register the native `sqb rename` and `sqb mv` command and its contract-edit helpers. -use std::cell::RefCell; -use std::path::{Path, PathBuf}; +use pyo3::prelude::{Bound, PyModule, PyResult}; -use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use serde::Serialize; -use serde::de::DeserializeOwned; -use serde_json::json; -use sqlbuild_refactor::refactoring::errors::{RefactorError, RefactorErrorKind}; -use sqlbuild_refactor::refactoring::main::commit_refactor_plan::commit_refactor_plan; -use sqlbuild_refactor::refactoring::main::plan_refactor::plan_refactor; -use sqlbuild_refactor::refactoring::main::stage_refactor_plan::stage_refactor_plan; -use sqlbuild_refactor::refactoring::main::with_model_migration::with_model_migration; -use sqlbuild_refactor::refactoring::models::{ - DeclarationPlacement, ModelFacts, RefactorFacts, RefactorPlan, RefactorRequest, +use crate::bindings::_helpers::refactor::{ + contract_compare, contract_edits, python_strings, refactor_command, }; -use sqlbuild_refactor::refactoring::types::Originals; - -use crate::bindings::_helpers::boundary::panics::{ - NativeCompilerError, compiler_guard, value_error, -}; - -fn decode(text: &str) -> PyResult { - serde_json::from_str(text).map_err(value_error) -} - -fn encode(key: &str, result: Result) -> PyResult { - let value = match result { - Ok(value) => json!({ key: value }), - Err(error) if error.kind == RefactorErrorKind::Internal => { - return Err(NativeCompilerError::new_err(error.message)); - } - Err(error) => json!({ "error": error }), - }; - serde_json::to_string(&value).map_err(value_error) -} - -/// Plan one refactoring; `declaration_placement((model, destination))` returns JSON placement. -#[pyfunction] -fn plan_refactor_json( - py: Python<'_>, - facts_json: &str, - request_json: &str, - declaration_placement: Py, -) -> PyResult { - compiler_guard(|| { - let facts: RefactorFacts = decode(facts_json)?; - let request: RefactorRequest = decode(request_json)?; - let host_error: RefCell> = RefCell::new(None); - let host = |model: &str, destination: &str| { - let placement = declaration_placement - .call1(py, ((model, destination),)) - .and_then(|value| value.extract::(py)) - .and_then(|text| decode::(&text)); - placement.map_err(|error| { - host_error.replace(Some(error)); - RefactorError::internal("the declaration-move host failed") - }) - }; - let result = plan_refactor(&facts, &request, &host); - if let Some(error) = host_error.into_inner() { - return Err(error); - } - encode("plan", result) - }) -} - -/// Copy the project inputs once, apply the plan in `staging_dir`, and return the originals. -#[pyfunction] -fn stage_refactor_plan_json( - project_dir: &str, - staging_dir: &str, - plan_json: &str, - copy_inputs: bool, -) -> PyResult { - compiler_guard(|| { - let plan: RefactorPlan = decode(plan_json)?; - encode( - "originals", - stage_refactor_plan( - Path::new(project_dir), - Path::new(staging_dir), - &plan, - copy_inputs, - ), - ) - }) -} - -/// The plan with model `migrate_from` added whenever the relation moves. -#[pyfunction] -#[pyo3(signature = (plan_json, before_json, after_json, originals_json))] -fn with_model_migration_json( - plan_json: &str, - before_json: &str, - after_json: Option<&str>, - originals_json: &str, -) -> PyResult { - compiler_guard(|| { - let plan: RefactorPlan = decode(plan_json)?; - let before: Vec = decode(before_json)?; - let after: Option> = after_json.map(decode).transpose()?; - let originals: Originals = decode(originals_json)?; - encode( - "plan", - Ok(with_model_migration( - &plan, - &before, - after.as_deref(), - &originals, - )), - ) - }) -} - -/// Write a verified plan into the project, restoring every file if any write fails. -#[pyfunction] -fn commit_refactor_plan_json( - project_dir: &str, - originals_json: &str, - plan_json: &str, -) -> PyResult { - compiler_guard(|| { - let plan: RefactorPlan = decode(plan_json)?; - let originals: Originals = decode(originals_json)?; - let written = - commit_refactor_plan(Path::new(project_dir), &originals, &plan).map(path_strings); - encode("written", written) - }) -} - -fn path_strings(paths: Vec) -> Vec { - paths - .into_iter() - .map(|path| path.to_string_lossy().into_owned()) - .collect() -} pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_refactor_json, module)?)?; - module.add_function(wrap_pyfunction!(stage_refactor_plan_json, module)?)?; - module.add_function(wrap_pyfunction!(with_model_migration_json, module)?)?; - module.add_function(wrap_pyfunction!(commit_refactor_plan_json, module)?)?; - crate::bindings::_helpers::refactor::contract_edits::register(module)?; - crate::bindings::_helpers::refactor::contract_compare::register(module)?; - crate::bindings::_helpers::refactor::python_strings::register(module)?; - Ok(()) + refactor_command::register(module)?; + contract_edits::register(module)?; + contract_compare::register(module)?; + python_strings::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs new file mode 100644 index 0000000000..41c6b55dab --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs @@ -0,0 +1,250 @@ +//! Append canonical integration-result records for a parent integration, one per claimed terminal. + +use std::sync::Mutex; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::integration_results::errors::IntegrationRecordError; +use sqlbuild_core::terminal_events::models::TerminalRequest; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::result_output::integration_records::{ + clone_record, degraded_record, degraded_warning, execution_succeeded, integration_record, +}; +use crate::bindings::_helpers::result_output::integration_values::Checked; +use crate::bindings::_helpers::result_output::projection::resource_identity; +use crate::bindings::_helpers::result_output::terminal_index::{ + TerminalEventClaim, TerminalEventIndex, current_index_object, +}; + +const RESULT_PATH_ENV: &str = "SQLBUILD_INTEGRATION_RESULT_PATH"; +const BUILD_OPTIONS: (&[&str], &[&str]) = (&["result", "plan"], &["command"]); + +/// Thread-safe JSON Lines execution event writer. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ExecutionEventWriter { + /// A `threading.Lock`; it releases the GIL while it waits. + lock: Py, + terminal_index: Option>, + /// The open text stream, until the writer is closed. + stream: Mutex>>, +} + +/// A Python context manager entered for the duration of `body`. +fn within(manager: &Bound<'_, PyAny>, body: impl FnOnce() -> PyResult) -> PyResult { + manager.call_method0("__enter__")?; + let outcome: PyResult = body(); + let none = manager.py().None(); + manager.call_method1("__exit__", (&none, &none, &none))?; + outcome +} + +impl ExecutionEventWriter { + fn open_stream(&self, py: Python<'_>) -> PyResult>> { + Ok(self + .stream + .lock() + .map_err(|_| PyRuntimeError::new_err("execution event writer lock is poisoned"))? + .as_ref() + .map(|stream| stream.clone_ref(py))) + } + + /// Run `body` holding the writer lock and the index's output scope, when the stream is open. + fn serialized( + &self, + py: Python<'_>, + body: impl FnOnce(&Bound<'_, PyAny>) -> PyResult<()>, + ) -> PyResult<()> { + within(self.lock.bind(py), || { + let Some(stream) = self.open_stream(py)? else { + return Ok(()); + }; + match &self.terminal_index { + Some(index) => within(&index.get().output_lock(py), || body(stream.bind(py))), + None => body(stream.bind(py)), + } + }) + } + + fn claim( + &self, + py: Python<'_>, + request: &TerminalRequest, + ) -> PyResult> { + match &self.terminal_index { + Some(index) => index.get().claim(py, request), + None => Ok(None), + } + } + + /// Write the record, or after a contract failure warn and write the degraded record. + fn publish( + stream: &Bound<'_, PyAny>, + terminal: &Bound<'_, PyAny>, + record: Checked>, + degraded: impl FnOnce() -> Checked>, + ) -> PyResult<()> { + let error: IntegrationRecordError = match record? { + Ok(Some(line)) => return write(stream, &line), + Ok(None) => return Ok(()), + Err(error) => error, + }; + let stderr: Bound<'_, PyAny> = stream.py().import("sys")?.getattr("stderr")?; + stderr.call_method1( + "write", + (degraded_warning(&error, execution_succeeded(terminal)?),), + )?; + stderr.call_method0("flush")?; + match degraded()? { + Ok(Some(line)) => write(stream, &line), + Ok(None) | Err(_) => Ok(()), + } + } +} + +fn write(stream: &Bound<'_, PyAny>, line: &str) -> PyResult<()> { + stream.call_method1("write", (line,))?; + stream.call_method0("flush")?; + Ok(()) +} + +#[pymethods] +impl ExecutionEventWriter { + #[new] + #[pyo3(signature = (*, path = None))] + fn new(py: Python<'_>, path: Option>) -> PyResult { + let configured: Bound<'_, PyAny> = py + .import("os")? + .getattr("environ")? + .call_method1("get", (RESULT_PATH_ENV,))?; + let path: Option> = + match path.filter(|path| path.is_truthy().unwrap_or(true)) { + Some(path) => Some(path), + None if configured.is_truthy()? => Some( + py.import("pathlib")? + .getattr("Path")? + .call1((configured,))?, + ), + None => None, + }; + let stream: Option> = match path { + Some(path) => { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("parents", true)?; + keywords.set_item("exist_ok", true)?; + path.getattr("parent")? + .call_method("mkdir", (), Some(&keywords))?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("mode", "a")?; + options.set_item("encoding", "utf-8")?; + Some(path.call_method("open", (), Some(&options))?.unbind()) + } + None => None, + }; + Ok(Self { + lock: py.import("threading")?.call_method0("Lock")?.unbind(), + terminal_index: current_index_object(py)?, + stream: Mutex::new(stream), + }) + } + + /// Write canonical terminal evidence plus typed build-result enrichment. + #[pyo3(signature = (**options))] + fn write_build_result( + &self, + py: Python<'_>, + options: Option>, + ) -> PyResult<()> { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "write_build_result", options, BUILD_OPTIONS)?; + let result: Bound<'_, PyAny> = arguments.value("result")?; + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let command: String = if arguments.passed("command")? { + arguments.value("command")?.extract()? + } else { + "build".to_owned() + }; + self.serialized(py, |stream| { + let Some((resource_name, resource_id)) = resource_identity(&result)? else { + return Ok(()); + }; + let Some(claim) = self.claim( + py, + &TerminalRequest { + resource_name, + resource_id: Some(resource_id), + ..TerminalRequest::default() + }, + )? + else { + return Ok(()); + }; + let terminal: Bound<'_, PyAny> = claim.terminal_event(py); + let sequence: usize = claim.sequence(); + Self::publish( + stream, + &terminal, + integration_record((&result, &plan), (&terminal, sequence), &command), + || degraded_record(&result, (&terminal, sequence), &command), + ) + }) + } + + /// Write canonical terminal evidence plus typed clone-result enrichment. + #[pyo3(signature = (*, item, resource_type))] + fn write_clone_result( + &self, + py: Python<'_>, + item: Bound<'_, PyAny>, + resource_type: &str, + ) -> PyResult<()> { + self.serialized(py, |stream| { + let name: String = item.getattr("name")?.extract()?; + let Some(claim) = self.claim( + py, + &TerminalRequest { + resource_id: Some(format!("{resource_type}:{name}")), + resource_name: name, + ..TerminalRequest::default() + }, + )? + else { + return Ok(()); + }; + let terminal: Bound<'_, PyAny> = claim.terminal_event(py); + let sequence: usize = claim.sequence(); + Self::publish( + stream, + &terminal, + Ok(clone_record(&item, (&terminal, sequence))?.map(Some)), + || degraded_record(&item, (&terminal, sequence), "clone"), + ) + }) + } + + /// Close the event stream when configured. + fn close(&self, py: Python<'_>) -> PyResult<()> { + within(self.lock.bind(py), || { + let Some(stream) = self.open_stream(py)? else { + return Ok(()); + }; + let close = || -> PyResult<()> { + *self.stream.lock().map_err(|_| { + PyRuntimeError::new_err("execution event writer lock is poisoned") + })? = None; + stream.bind(py).call_method0("close")?; + Ok(()) + }; + match &self.terminal_index { + Some(index) => within(&index.get().output_lock(py), close), + None => close(), + } + }) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs new file mode 100644 index 0000000000..60fdca0baf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs @@ -0,0 +1,193 @@ +//! Decoded integration-result records, read by the integrations that consume them. + +use pyo3::exceptions::PyAttributeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::integration_results::errors::IntegrationRecordError; +use sqlbuild_core::integration_results::main::encode_integration_value::encode_integration_value; +use sqlbuild_core::integration_results::main::integration_record_fits::integration_record_fits; +use sqlbuild_core::integration_results::main::normalize_integration_record::normalize_integration_record; +use sqlbuild_core::integration_results::main::strip_optional_enrichment::strip_optional_enrichment; +use sqlbuild_core::json::models::JsonValue; + +use crate::bindings::_helpers::result_output::integration_values::{json_value, validation_error}; +use crate::bindings::_helpers::scopes::scope_report_objects::json_to_python; + +/// One decoded part of a record: its fields as attributes, and as a dictionary. +fn field(fields: &Py, py: Python<'_>, name: &str) -> PyResult> { + match fields.bind(py).get_item(name)? { + Some(value) => Ok(value.unbind()), + None => Err(PyAttributeError::new_err(name.to_owned())), + } +} + +fn copied(fields: &Py, py: Python<'_>) -> PyResult> { + Ok(fields.bind(py).copy()?.unbind()) +} + +/// Asset enrichment of one integration result. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct IntegrationAssetResult { + fields: Py, +} + +/// Check enrichment of one integration result. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct IntegrationCheckResult { + fields: Py, +} + +/// One validated integration-result record. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct IntegrationResultEnvelope { + record: JsonValue, + fields: Py, + asset: Option>, + checks: Py, +} + +#[pymethods] +impl IntegrationAssetResult { + fn __getattr__(&self, py: Python<'_>, name: &str) -> PyResult> { + field(&self.fields, py, name) + } + + /// Every field, as `dataclasses.asdict` gave it. + fn as_dict(&self, py: Python<'_>) -> PyResult> { + copied(&self.fields, py) + } +} + +#[pymethods] +impl IntegrationCheckResult { + fn __getattr__(&self, py: Python<'_>, name: &str) -> PyResult> { + field(&self.fields, py, name) + } + + /// Every field, as `dataclasses.asdict` gave it. + fn as_dict(&self, py: Python<'_>) -> PyResult> { + copied(&self.fields, py) + } +} + +fn parsed(py: Python<'_>, line: &str) -> PyResult> { + let json: Bound<'_, PyModule> = py.import("json")?; + let payload: Bound<'_, PyAny> = match json.call_method1("loads", (line,)) { + Ok(payload) => payload, + Err(error) if error.is_instance(py, &json.getattr("JSONDecodeError")?) => { + return Ok(Err(IntegrationRecordError { + message: "malformed integration result JSON".to_owned(), + })); + } + Err(error) => return Err(error), + }; + Ok(json_value(&payload)?.ok_or_else(|| IntegrationRecordError { + message: "integration result must be a JSON object".to_owned(), + })) +} + +fn decoded(py: Python<'_>, record: JsonValue) -> PyResult { + let normalized: Vec<(String, JsonValue)> = + normalize_integration_record(record).map_err(|error| validation_error(py, &error))?; + let fields: Bound<'_, PyDict> = PyDict::new(py); + let mut asset: Option> = None; + let mut checks: Vec> = Vec::new(); + for (key, value) in normalized.iter().cloned() { + match (key.as_str(), value) { + ("asset", JsonValue::Object(entries)) => { + let fields: Py = json_to_python(py, JsonValue::Object(entries), false)?; + asset = Some(Py::new( + py, + IntegrationAssetResult { + fields: fields.extract(py)?, + }, + )?); + } + ("asset", _) => {} + ("checks", JsonValue::Array(items)) => { + for item in items { + let fields: Py = json_to_python(py, item, false)?; + checks.push(Py::new( + py, + IntegrationCheckResult { + fields: fields.extract(py)?, + }, + )?); + } + } + (_, value) => fields.set_item(key, json_to_python(py, value, false)?)?, + } + } + Ok(IntegrationResultEnvelope { + record: JsonValue::Object(normalized), + fields: fields.unbind(), + asset, + checks: PyTuple::new(py, checks)?.unbind(), + }) +} + +#[pymethods] +impl IntegrationResultEnvelope { + /// Decode and validate one integration result JSON record. + #[staticmethod] + fn from_json(py: Python<'_>, value: &Bound<'_, PyString>) -> PyResult { + let line: String = value.to_string_lossy().into_owned(); + if !integration_record_fits(line.len()) { + return Err(validation_error( + py, + &IntegrationRecordError { + message: "integration result record exceeds size limit".to_owned(), + }, + )); + } + let record: JsonValue = parsed(py, &line)?.map_err(|error| validation_error(py, &error))?; + decoded(py, record) + } + + /// Decode a record after dropping forward-compatible optional enrichment. + #[staticmethod] + fn from_json_without_optional_enrichment( + py: Python<'_>, + value: &Bound<'_, PyString>, + ) -> PyResult { + let line: String = value.to_string_lossy().into_owned(); + let record: JsonValue = parsed(py, &line)?.map_err(|error| validation_error(py, &error))?; + if !matches!(record, JsonValue::Object(_)) { + return Err(validation_error( + py, + &IntegrationRecordError { + message: "integration result must be a JSON object".to_owned(), + }, + )); + } + decoded(py, strip_optional_enrichment(record)) + } + + /// Encode the validated record as one JSON Lines record. + fn to_json(&self, py: Python<'_>) -> PyResult { + encode_integration_value(&self.record, true) + .map(|line| line + "\n") + .map_err(|error| validation_error(py, &error)) + } + + #[getter] + fn asset(&self, py: Python<'_>) -> Option> { + self.asset.as_ref().map(|asset| asset.clone_ref(py)) + } + + #[getter] + fn checks(&self, py: Python<'_>) -> Py { + self.checks.clone_ref(py) + } + + fn __getattr__(&self, py: Python<'_>, name: &str) -> PyResult> { + field(&self.fields, py, name) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs new file mode 100644 index 0000000000..de5bc7aa13 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs @@ -0,0 +1,600 @@ +//! Integration-result records built from a claimed terminal and its executor result. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyFloat, PyInt}; +use sqlbuild_core::integration_results::errors::IntegrationRecordError; +use sqlbuild_core::integration_results::main::encode_integration_value::encode_integration_value; +use sqlbuild_core::integration_results::main::normalize_integration_record::normalize_integration_record; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; +use crate::bindings::_helpers::plan_output::values::items; +use crate::bindings::_helpers::result_output::checks::{ + audit_passed, measurement_evidence, measurement_thresholds, python_check_status, +}; +use crate::bindings::_helpers::result_output::integration_values::{ + Checked, bounded_metadata, optional_text, required_json, text, text_or_empty, +}; +use crate::bindings::_helpers::result_output::projection::{ + ResultKind, audit_check_id, result_kind, sql_test_check_id, +}; + +const FAILED_EVENT: &str = "resource_attempt_failed"; +const SKIPPED_EVENT: &str = "resource_attempt_skipped"; +const PASS_OUTCOME: &str = "pass"; +const MICROBATCH_FIELDS: [(&str, &str); 19] = [ + ("strategy", "microbatch_strategy"), + ("reason", "microbatch_plan_reason"), + ("batch_count", "batch_count"), + ("batch_size", "batch_size"), + ("recovery_batch_count", "microbatch_recovery_batch_count"), + ("known_gap_count", "microbatch_known_gap_count"), + ( + "unaccounted_interval_count", + "microbatch_unaccounted_interval_count", + ), + ( + "synthetic_completion_count", + "microbatch_synthetic_completion_count", + ), + ( + "unknown_fingerprint_count", + "microbatch_unknown_fingerprint_count", + ), + ("contiguous_frontier", "microbatch_contiguous_frontier"), + ( + "unaccounted_partition_policy", + "microbatch_unaccounted_partition_policy", + ), + ("replay_requirement_id", "microbatch_replay_requirement_id"), + ( + "required_model_version_hash", + "microbatch_required_model_version_hash", + ), + ( + "physical_generation_id", + "microbatch_physical_generation_id", + ), + ("concurrent_enabled", "microbatch_concurrent_enabled"), + ("batch_concurrency", "microbatch_batch_concurrency"), + ("global_concurrency", "microbatch_global_concurrency"), + ( + "replay_requirement_state", + "microbatch_replay_requirement_state", + ), + ("limit", "microbatch_limit"), +]; + +type Entries = Vec<(String, JsonValue)>; + +/// Entries from Python values, failing the contract on a non-JSON value. +fn entries(values: Vec<(&str, Bound<'_, PyAny>)>) -> Checked { + let mut converted: Entries = Vec::with_capacity(values.len()); + for (key, value) in values { + match required_json(&value)? { + Ok(value) => converted.push((key.to_owned(), value)), + Err(error) => return Ok(Err(error)), + } + } + Ok(Ok(converted)) +} + +fn value<'py>(result: &Bound<'py, PyAny>, field: &str) -> PyResult> { + result.getattr(field)?.getattr("value") +} + +/// `.value` when it is truthy, else `None`. +fn truthy_value<'py>(result: &Bound<'py, PyAny>, field: &str) -> PyResult> { + let value: Bound<'py, PyAny> = result.getattr(field)?; + if !value.is_truthy()? { + return Ok(result.py().None().into_bound(result.py())); + } + value.getattr("value") +} + +/// `plan.[name].qualified_name` when the plan locates the resource. +fn planned_target<'py>( + plan: &Bound<'py, PyAny>, + field: &str, + name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = plan.py(); + if plan.is_none() { + return Ok(py.None().into_bound(py)); + } + let locations: Bound<'py, PyAny> = plan.getattr(field)?; + if !locations.contains(name)? { + return Ok(py.None().into_bound(py)); + } + locations.get_item(name)?.getattr("qualified_name") +} + +fn model_microbatch(result: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = result.py(); + let run_type: Bound<'_, PyAny> = result.getattr("microbatch_run_type")?; + if run_type.is_none() { + return Ok(JsonValue::Object(Vec::new())); + } + let values: Bound<'_, pyo3::types::PyDict> = pyo3::types::PyDict::new(py); + pyo3::types::PyDictMethods::set_item(&values, "run_type", run_type)?; + for (key, field) in MICROBATCH_FIELDS { + let item: Bound<'_, PyAny> = result.getattr(field)?; + if !item.is_none() { + pyo3::types::PyDictMethods::set_item(&values, key, item)?; + } + } + let count: Bound<'_, PyAny> = result.getattr("microbatch_limit_count")?; + if !count.is_none() { + pyo3::types::PyDictMethods::set_item(&values, "count", count)?; + } + let action: Bound<'_, PyAny> = result.getattr("microbatch_limit_action")?; + if !action.is_none() { + pyo3::types::PyDictMethods::set_item(&values, "action", action.getattr("value")?)?; + } + bounded_metadata(values.as_any()) +} + +/// `_asset_result`: the asset enrichment of an asset-producing result. +fn asset<'py>(result: &Bound<'py, PyAny>, plan: &Bound<'py, PyAny>) -> Checked> { + let py: Python<'py> = result.py(); + let none = || py.None().into_bound(py); + let values: Vec<(&str, Bound<'py, PyAny>)> = match result_kind(result)? { + ResultKind::Model => { + let name: Bound<'py, PyAny> = result.getattr("model_name")?; + let planned: Bound<'py, PyAny> = planned_target(plan, "model_locations", &name)?; + let mut values: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("kind", "model".into_pyobject(py)?.into_any()), + ("name", name), + ("status", value(result, "status")?), + ( + "target", + if planned.is_truthy()? { + planned + } else { + result.getattr("promoted_relation")? + }, + ), + ("staging_relation", result.getattr("staging_relation")?), + ("failed_phase", truthy_value(result, "failed_phase")?), + ]; + let mut converted: Entries = match entries(std::mem::take(&mut values))? { + Ok(converted) => converted, + Err(error) => return Ok(Err(error)), + }; + for (key, evidence) in [ + ( + "future_cursor_safety", + future_cursor_safety(&result.getattr("future_cursor_safety")?)?, + ), + ( + "maximum_start_safety", + maximum_start_safety(&result.getattr("maximum_start_safety")?)?, + ), + ] { + let evidence: Bound<'py, PyAny> = if evidence.is_none() { + pyo3::types::PyDict::new(py).into_any() + } else { + evidence + }; + converted.push((key.to_owned(), bounded_metadata(&evidence)?)); + } + converted.push(("microbatch".to_owned(), model_microbatch(result)?)); + return Ok(Ok(Some(converted))); + } + ResultKind::Seed => { + let name: Bound<'py, PyAny> = result.getattr("seed_name")?; + vec![ + ("kind", "seed".into_pyobject(py)?.into_any()), + ("target", planned_target(plan, "seed_locations", &name)?), + ("name", name), + ("status", value(result, "status")?), + ] + } + ResultKind::Function => { + let name: Bound<'py, PyAny> = result.getattr("function_name")?; + vec![ + ("kind", result.getattr("function_kind")?), + ("target", planned_target(plan, "function_locations", &name)?), + ("name", name), + ("status", value(result, "status")?), + ] + } + ResultKind::Load => vec![ + ("kind", value(result, "resource_kind")?), + ("name", result.getattr("source_name")?), + ("status", value(result, "status")?), + ("target", result.getattr("target")?), + ("staging_relation", result.getattr("staging_relation")?), + ("loader", result.getattr("loader_name")?), + ("rows_loaded", result.getattr("rows_loaded")?), + ], + ResultKind::PythonNode => vec![ + ("kind", value(result, "kind")?), + ("name", result.getattr("node_name")?), + ("status", value(result, "status")?), + ("materialized", result.getattr("materialized")?), + ], + _ => { + let _ = none; + return Ok(Ok(None)); + } + }; + Ok(entries(values)?.map(Some)) +} + +fn audit_check<'py>(result: &Bound<'py, PyAny>) -> Checked { + let py: Python<'py> = result.py(); + let check_id: Bound<'py, PyAny> = audit_check_id(result)?.into_pyobject(py)?.into_any(); + let (evidence_count, evidence_truncated) = measurement_evidence(result)?; + entries(vec![ + ("kind", "audit".into_pyobject(py)?.into_any()), + ("name", result.getattr("audit_name")?), + ("check_id", check_id.clone()), + ("dag_check_id", check_id), + ( + "passed", + PyBool::new(py, audit_passed(result)?).to_owned().into_any(), + ), + ("status", value(result, "outcome")?), + ("severity", value(result, "severity")?), + ("row_count", result.getattr("row_count")?), + ("attachment_kind", value(result, "attachment_kind")?), + ("asset_name", result.getattr("attached_target_name")?), + ( + "attached_column_name", + result.getattr("attached_column_name")?, + ), + ( + "attached_target_name", + result.getattr("attached_target_name")?, + ), + ("run_scope_phase", value(result, "run_scope_phase")?), + ("reused", result.getattr("reused")?), + ("evaluation_mode", value(result, "evaluation_mode")?), + ("measured_value", result.getattr("measured_value")?), + ("sample_count", result.getattr("sample_count")?), + ("sample_unit", result.getattr("sample_unit")?), + ("minimum_samples", result.getattr("minimum_samples")?), + ( + "thresholds", + measurement_thresholds(&result.getattr("thresholds")?)?, + ), + ("evidence_count", evidence_count), + ("evidence_truncated", evidence_truncated), + ( + "audit_definition_name", + result.getattr("audit_definition_name")?, + ), + ("audit_description", result.getattr("audit_description")?), + ]) +} + +/// `_check_results`: the check enrichment of a check-producing result. +fn checks<'py>(result: &Bound<'py, PyAny>) -> Checked> { + let py: Python<'py> = result.py(); + let single = |checked: Checked| -> Checked> { + Ok(checked?.map(|check| vec![check])) + }; + match result_kind(result)? { + ResultKind::Audit => single(audit_check(result)), + ResultKind::SqlTest => { + let check_id: Bound<'py, PyAny> = + sql_test_check_id(result)?.into_pyobject(py)?.into_any(); + let steps: Vec> = items(&result.getattr("step_results")?)?; + let outcome: Bound<'py, PyAny> = value(result, "outcome")?; + single(entries(vec![ + ("kind", "sql_test".into_pyobject(py)?.into_any()), + ("name", result.getattr("test_name")?), + ("check_id", check_id.clone()), + ("dag_check_id", check_id), + ( + "passed", + PyBool::new(py, outcome.eq(PASS_OUTCOME)?) + .to_owned() + .into_any(), + ), + ("status", outcome), + ( + "asset_name", + match steps.first() { + Some(step) => step.getattr("model_name")?, + None => py.None().into_bound(py), + }, + ), + ])) + } + ResultKind::PythonCheck => { + let name: String = result.getattr("node_name")?.extract()?; + let check_id: Bound<'py, PyAny> = format!("check:{name}").into_pyobject(py)?.into_any(); + single(entries(vec![ + ("kind", "python_check".into_pyobject(py)?.into_any()), + ("name", result.getattr("node_name")?), + ("check_id", check_id.clone()), + ("dag_check_id", check_id), + ("passed", result.getattr("passed")?), + ( + "status", + python_check_status(result)?.into_pyobject(py)?.into_any(), + ), + ("severity", value(result, "severity")?), + ])) + } + _ => Ok(Ok(Vec::new())), + } +} + +/// The payload attempt number when it is an `int`, else 0. +fn attempt_number(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() && !value.is_instance_of::() { + let decimal: String = value.str()?.to_string(); + return Ok(JsonInteger::parse(&decimal).map_or(JsonValue::Null, JsonValue::Integer)); + } + Ok(JsonValue::Integer(JsonInteger::from(0_i64))) +} + +/// The payload duration when it is a number, `bool` included, else `null`. +fn duration(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(required_json(value)?.unwrap_or(JsonValue::Null)); + } + Ok(JsonValue::Null) +} + +/// The envelope fields every record copies from its terminal, in record order. +fn terminal_fields(terminal: &Bound<'_, PyAny>, event_sequence: usize) -> PyResult { + let payload: Bound<'_, PyAny> = terminal.getattr("payload")?; + let get = |key: &str| payload.call_method1("get", (key,)); + let attribute = |key: &str| -> PyResult { + Ok(required_json(&terminal.getattr(key)?)?.unwrap_or(JsonValue::Null)) + }; + let mut fields: Entries = vec![ + ( + "schema_version".to_owned(), + JsonValue::Integer(JsonInteger::from(1_i64)), + ), + ("record_kind".to_owned(), text("integration_result")), + ("event_id".to_owned(), attribute("event_id")?), + ( + "event_sequence".to_owned(), + JsonValue::Integer(JsonInteger::from(event_sequence as u64)), + ), + ("event_type".to_owned(), attribute("event_type")?), + ( + "occurred_at".to_owned(), + optional_text(&terminal.getattr("occurred_at")?.call_method0("isoformat")?)?, + ), + ("invocation_id".to_owned(), attribute("invocation_id")?), + ]; + for key in ["run_id", "resource_id", "resource_attempt_id"] { + fields.push((key.to_owned(), text_or_empty(&terminal.getattr(key)?)?)); + } + fields.push(("operation_id".to_owned(), attribute("operation_id")?)); + fields.push(("statement_id".to_owned(), attribute("statement_id")?)); + fields.push(( + "resource_kind".to_owned(), + text_or_empty(&get("resource_kind")?)?, + )); + fields.push(( + "resource_name".to_owned(), + text_or_empty(&get("resource_name")?)?, + )); + fields.push(( + "attempt_number".to_owned(), + attempt_number(&get("attempt_number")?)?, + )); + fields.push(("duration_ms".to_owned(), duration(&get("duration_ms")?)?)); + Ok(fields) +} + +fn failure_fields(terminal: &Bound<'_, PyAny>) -> PyResult { + let payload: Bound<'_, PyAny> = terminal.getattr("payload")?; + let mut fields: Entries = Vec::new(); + for key in ["error_code", "error_type", "skip_code", "skip_mode"] { + fields.push(( + key.to_owned(), + optional_text(&payload.call_method1("get", (key,))?)?, + )); + } + Ok(fields) +} + +/// `_canonical_status`: failed or skipped from the terminal, else the result's own status. +fn canonical_status(event_type: &str, fallback: JsonValue) -> JsonValue { + match event_type { + FAILED_EVENT => text("failed"), + SKIPPED_EVENT => text("skipped"), + _ => fallback, + } +} + +fn payload_text(terminal: &Bound<'_, PyAny>, key: &str) -> PyResult { + let value: JsonValue = + optional_text(&terminal.getattr("payload")?.call_method1("get", (key,))?)?; + Ok(match value { + JsonValue::Null => text(""), + other => other, + }) +} + +/// The complete record, encoded as one JSON Lines record with its newline. +fn encoded( + terminal: &Bound<'_, PyAny>, + event_sequence: usize, + (command, degraded): (&str, bool), + (asset, checks): (Option, Vec), +) -> Checked { + let mut record: Entries = terminal_fields(terminal, event_sequence)?; + let output_kind: &str = if asset.is_some() { "asset" } else { "check" }; + record.push(("output_kind".to_owned(), text(output_kind))); + record.push(("command".to_owned(), text(command))); + record.extend(failure_fields(terminal)?); + record.push(("projection_degraded".to_owned(), JsonValue::Bool(degraded))); + record.push(( + "asset".to_owned(), + asset.map_or(JsonValue::Null, JsonValue::Object), + )); + record.push(( + "checks".to_owned(), + JsonValue::Array(checks.into_iter().map(JsonValue::Object).collect()), + )); + let normalized: Entries = match normalize_integration_record(JsonValue::Object(record)) { + Ok(normalized) => normalized, + Err(error) => return Ok(Err(error)), + }; + Ok(encode_integration_value(&JsonValue::Object(normalized), true).map(|line| line + "\n")) +} + +fn event_type(terminal: &Bound<'_, PyAny>) -> PyResult { + terminal.getattr("event_type")?.extract() +} + +/// `build_integration_result`, encoded; `None` when the result carries no enrichment. +pub(crate) fn integration_record( + (result, plan): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (terminal, event_sequence): (&Bound<'_, PyAny>, usize), + command: &str, +) -> Checked> { + let mut asset: Option = match asset(result, plan)? { + Ok(asset) => asset, + Err(error) => return Ok(Err(error)), + }; + if let Some(entries) = &mut asset { + let event: String = event_type(terminal)?; + let kind: JsonValue = payload_text(terminal, "resource_kind")?; + let name: JsonValue = payload_text(terminal, "resource_name")?; + for (key, value) in entries.iter_mut() { + match key.as_str() { + "kind" => *value = kind.clone(), + "name" => *value = name.clone(), + "status" => { + *value = canonical_status(&event, std::mem::replace(value, JsonValue::Null)) + } + _ => {} + } + } + } + let checks: Vec = match checks(result)? { + Ok(checks) => checks, + Err(error) => return Ok(Err(error)), + }; + if asset.is_none() && checks.is_empty() { + return Ok(Ok(None)); + } + Ok(encoded(terminal, event_sequence, (command, false), (asset, checks))?.map(Some)) +} + +/// `build_clone_integration_result`, encoded. +pub(crate) fn clone_record( + item: &Bound<'_, PyAny>, + (terminal, event_sequence): (&Bound<'_, PyAny>, usize), +) -> Checked { + let event: String = event_type(terminal)?; + let values: Entries = match entries(vec![ + ("action", value(item, "action")?), + ("target", item.getattr("destination_relation")?), + ("origin_relation", item.getattr("origin_relation")?), + ])? { + Ok(values) => values, + Err(error) => return Ok(Err(error)), + }; + let mut asset: Entries = vec![ + ( + "kind".to_owned(), + payload_text_or_str(terminal, "resource_kind")?, + ), + ( + "name".to_owned(), + payload_text_or_str(terminal, "resource_name")?, + ), + ( + "status".to_owned(), + canonical_status( + &event, + required_json(&value(item, "status")?)?.unwrap_or(JsonValue::Null), + ), + ), + ]; + asset.extend(values); + encoded( + terminal, + event_sequence, + ("clone", false), + (Some(asset), Vec::new()), + ) +} + +/// `str(payload.get(key) or "")`, as the clone asset's identity. +fn payload_text_or_str(terminal: &Bound<'_, PyAny>, key: &str) -> PyResult { + text_or_empty(&terminal.getattr("payload")?.call_method1("get", (key,))?) +} + +/// `build_degraded_integration_result`, encoded; `None` for results without enrichment. +pub(crate) fn degraded_record( + result: &Bound<'_, PyAny>, + (terminal, event_sequence): (&Bound<'_, PyAny>, usize), + command: &str, +) -> Checked> { + let event: String = event_type(terminal)?; + let kind: JsonValue = payload_text_or_str(terminal, "resource_kind")?; + let name: JsonValue = payload_text_or_str(terminal, "resource_name")?; + let (asset, checks): (Option, Vec) = match result_kind(result)? { + ResultKind::Model + | ResultKind::Seed + | ResultKind::Function + | ResultKind::Load + | ResultKind::PythonNode + | ResultKind::CloneItem => { + let status: &str = match event.as_str() { + FAILED_EVENT => "failed", + SKIPPED_EVENT => "skipped", + _ => "success", + }; + ( + Some(vec![ + ("kind".to_owned(), kind), + ("name".to_owned(), name), + ("status".to_owned(), text(status)), + ]), + Vec::new(), + ) + } + ResultKind::Audit | ResultKind::SqlTest | ResultKind::PythonCheck => { + let failed: bool = event == FAILED_EVENT; + let check_id: JsonValue = text_or_empty(&terminal.getattr("resource_id")?)?; + ( + None, + vec![vec![ + ("kind".to_owned(), kind), + ("name".to_owned(), name), + ("check_id".to_owned(), check_id), + ("passed".to_owned(), JsonValue::Bool(!failed)), + ( + "status".to_owned(), + text(if failed { "error" } else { "pass" }), + ), + ]], + ) + } + ResultKind::Other => return Ok(Ok(None)), + }; + Ok(encoded(terminal, event_sequence, (command, true), (asset, checks))?.map(Some)) +} + +/// Whether the record failure should not hide a completed warehouse execution. +pub(crate) fn execution_succeeded(terminal: &Bound<'_, PyAny>) -> PyResult { + Ok(event_type(terminal)?.ends_with("completed")) +} + +/// The warning written before a degraded record. +pub(crate) fn degraded_warning(error: &IntegrationRecordError, succeeded: bool) -> String { + let completion: &str = if succeeded { + "warehouse execution completed successfully" + } else { + "execution result remains authoritative" + }; + format!( + "warning: integration output degraded: {}; {completion}\n", + error.message + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs new file mode 100644 index 0000000000..b98ab9a191 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs @@ -0,0 +1,114 @@ +//! Python values as integration JSON, and contract failures as `ObservabilityValidationError`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, + PyStringMethods, PyTuple, +}; +use sqlbuild_core::integration_results::errors::IntegrationRecordError; +use sqlbuild_core::integration_results::main::encode_integration_value::encode_integration_value; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const OBSERVABILITY_ERRORS: &str = "sqlbuild.runtime.observability.exceptions"; +const VALIDATION_ERROR: &str = "ObservabilityValidationError"; + +/// A contract failure, or a Python failure while reading the values. +pub(crate) type Checked = PyResult>; + +/// The `ObservabilityValidationError` Python raised for the same contract failure. +pub(crate) fn validation_error(py: Python<'_>, error: &IntegrationRecordError) -> PyErr { + match imported(py, OBSERVABILITY_ERRORS, VALIDATION_ERROR) + .and_then(|class| class.call1((error.message.as_str(),))) + { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// A plain JSON Python value, or `None` when it holds something JSON cannot carry. +pub(crate) fn json_value(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Some(JsonValue::Null)); + } + if let Ok(flag) = value.downcast::() { + return Ok(Some(JsonValue::Bool(flag.is_true()))); + } + if value.is_instance_of::() { + let decimal: String = value.str()?.to_string(); + return Ok(JsonInteger::parse(&decimal).map(JsonValue::Integer)); + } + if value.is_instance_of::() { + return Ok(Some(JsonValue::Float(value.extract()?))); + } + if let Ok(text) = value.downcast::() { + return Ok(Some(JsonValue::String(text.to_string_lossy().into_owned()))); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + let Some(item) = json_value(&item?)? else { + return Ok(None); + }; + items.push(item); + } + return Ok(Some(JsonValue::Array(items))); + } + if let Ok(mapping) = value.downcast::() { + let mut entries: Vec<(String, JsonValue)> = Vec::new(); + for (key, item) in mapping.iter() { + let Ok(key) = key.downcast::() else { + return Ok(None); + }; + let Some(item) = json_value(&item)? else { + return Ok(None); + }; + entries.push((key.to_string_lossy().into_owned(), item)); + } + return Ok(Some(JsonValue::Object(entries))); + } + Ok(None) +} + +/// A plain JSON Python value, or the contract failure of a non-JSON value. +pub(crate) fn required_json(value: &Bound<'_, PyAny>) -> Checked { + Ok(json_value(value)?.ok_or_else(|| IntegrationRecordError { + message: "integration value is not JSON".to_owned(), + })) +} + +/// `_bounded_metadata`: the value when it encodes within the metadata bounds, else `{}`. +pub(crate) fn bounded_metadata(value: &Bound<'_, PyAny>) -> PyResult { + let empty: JsonValue = JsonValue::Object(Vec::new()); + let Some(converted) = json_value(value)? else { + return Ok(empty); + }; + if encode_integration_value(&converted, false).is_err() { + return Ok(empty); + } + Ok(converted) +} + +/// `value` when it is a string, else `null`. +pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(text) = value.downcast::() { + return Ok(JsonValue::String(text.to_string_lossy().into_owned())); + } + Ok(JsonValue::Null) +} + +/// `str(value or "")`. +pub(crate) fn text_or_empty(value: &Bound<'_, PyAny>) -> PyResult { + if !value.is_truthy()? { + return Ok(JsonValue::String(String::new())); + } + Ok(JsonValue::String( + value.str()?.to_string_lossy().into_owned(), + )) +} + +/// A JSON string. +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs index 6b6832cd45..1e2e9b28c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs @@ -1,6 +1,10 @@ pub(crate) mod assets; pub(crate) mod checks; pub(crate) mod documents; +pub(crate) mod event_writer; +pub(crate) mod integration_envelope; +pub(crate) mod integration_records; +pub(crate) mod integration_values; pub(crate) mod projection; pub(crate) mod registration; pub(crate) mod terminal_index; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs index b5e1838c07..890cf21372 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs @@ -2,9 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::result_output::{documents, terminal_index}; +use crate::bindings::_helpers::result_output::{ + documents, event_writer, integration_envelope, terminal_index, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { terminal_index::register(module)?; + integration_envelope::register(module)?; + event_writer::register(module)?; documents::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs index 3bc2a17894..5df6e3d505 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs @@ -265,6 +265,25 @@ pub(crate) fn current_index(py: Python<'_>) -> PyResult) -> PyResult>> { + let installed: Bound<'_, PyAny> = context_variable(py)?.call_method0("get")?; + if installed.is_none() { + return Ok(None); + } + Ok(Some(installed.extract()?)) +} + +impl TerminalEventClaim { + pub(crate) fn terminal_event<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + self.terminal.bind(py).clone() + } + + pub(crate) fn sequence(&self) -> usize { + self.event_sequence + } +} + /// Return the terminal event index installed for this invocation. #[pyfunction] fn current_terminal_event_index(py: Python<'_>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs new file mode 100644 index 0000000000..922c6cede2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs @@ -0,0 +1,131 @@ +//! Custom-rule import closures, implementation fingerprints and statically found test cases. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_pysource::rule_evidence::main::implementation_fingerprint::implementation_fingerprint; +use sqlbuild_pysource::rule_evidence::main::import_closure::import_closure as import_closure_of; +use sqlbuild_pysource::rule_evidence::main::test_evidence::test_evidence; +use sqlbuild_pysource::rule_evidence::models::RuleTestEvidence; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const PATHLIB_MODULE: &str = "pathlib"; +const INSPECT_MODULE: &str = "inspect"; +const HASHLIB_MODULE: &str = "hashlib"; + +fn python_version(py: Python<'_>) -> (u8, u8) { + let version = py.version_info(); + (version.major, version.minor) +} + +/// `Path(rule.source).resolve()` as a Python path, or `None` for a rule without a source file. +fn resolved_source<'py>(rule: &Bound<'py, PyAny>) -> PyResult>> { + let source: Bound<'py, PyAny> = rule.getattr("source")?; + if source.is_none() { + return Ok(None); + } + let path: Bound<'py, PyAny> = imported(rule.py(), PATHLIB_MODULE, "Path")?.call1((source,))?; + path.call_method0("resolve").map(Some) +} + +fn native_path(path: &Bound<'_, PyAny>) -> PyResult { + path.extract() +} + +fn closure_paths<'py>(py: Python<'py>, paths: Vec) -> PyResult> { + let class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; + let paths: Vec> = paths + .into_iter() + .map(|path| class.call1((path,))) + .collect::>()?; + PyTuple::new(py, paths) +} + +/// Repository-owned helper files imported by one rule module. +#[pyfunction] +#[pyo3(signature = (*, rule, project_dir))] +fn custom_rule_import_closure<'py>( + rule: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = rule.py(); + let Some(source) = resolved_source(rule)? else { + return Ok(PyTuple::empty(py)); + }; + let root: PathBuf = native_path(&project_dir.call_method0("resolve")?)?; + let closure: Vec = + import_closure_of(&native_path(&source)?, &root, python_version(py)); + closure_paths(py, closure) +} + +/// One rule's source file and helper closure fingerprint, or its check's source digest. +#[pyfunction] +#[pyo3(signature = (*, rule, project_dir, import_closure = None))] +fn custom_rule_implementation_fingerprint( + rule: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, + import_closure: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let py: Python<'_> = rule.py(); + let Some(source) = resolved_source(rule)? else { + let text: Bound<'_, PyAny> = + imported(py, INSPECT_MODULE, "getsource")?.call1((rule.getattr("check")?,))?; + return imported(py, HASHLIB_MODULE, "sha256")? + .call1((text.call_method0("encode")?,))? + .call_method0("hexdigest")? + .extract(); + }; + let root: PathBuf = native_path(&project_dir.call_method0("resolve")?)?; + let source: PathBuf = native_path(&source)?; + let closure: Vec = match import_closure { + Some(closure) if !closure.is_none() => closure + .try_iter()? + .map(|path| native_path(&path?)) + .collect::>()?, + _ => import_closure_of(&source, &root, python_version(py)), + }; + Ok(implementation_fingerprint(&source, &closure, &root)?) +} + +/// `(path, line, column, owner)` of each public-harness case that statically evaluates the rule. +#[pyfunction] +#[pyo3(signature = (*, rule, project_dir))] +fn custom_rule_test_evidence<'py>( + rule: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = rule.py(); + let Some(source) = resolved_source(rule)? else { + return Ok(Vec::new()); + }; + let root_path: Bound<'py, PyAny> = project_dir.call_method0("resolve")?; + let check_name: String = match rule.getattr("check")?.getattr("__name__") { + Ok(name) => name.extract()?, + Err(_) => String::new(), + }; + let Some(evidence) = test_evidence( + (&native_path(&source)?, &check_name), + (&native_path(project_dir)?, &native_path(&root_path)?), + python_version(py), + ) else { + source.call_method1("relative_to", (root_path,))?; + return Ok(Vec::new()); + }; + Ok(evidence + .into_iter() + .map(|item: RuleTestEvidence| (item.path, item.line, item.column, item.owner)) + .collect()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(custom_rule_import_closure, module)?)?; + module.add_function(wrap_pyfunction!( + custom_rule_implementation_fingerprint, + module + )?)?; + module.add_function(wrap_pyfunction!(custom_rule_test_evidence, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs new file mode 100644 index 0000000000..bd153bbdd6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs @@ -0,0 +1,438 @@ +//! DSL header scanning, header lint rules and header formatting fixes over Python lint models. + +use std::cell::RefCell; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::dsl_headers::constants::RULE_HEADER_PARSE; +use sqlbuild_rules::dsl_headers::main::format_headers::format_headers; +use sqlbuild_rules::dsl_headers::main::inner_header_text::inner_header_text; +use sqlbuild_rules::dsl_headers::main::lint_headers::lint_headers; +use sqlbuild_rules::dsl_headers::main::measurement_body_ranges::measurement_body_ranges as measurement_ranges; +use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers as scan; +use sqlbuild_rules::dsl_headers::main::sql_body_ranges::sql_body_ranges as body_ranges; +use sqlbuild_rules::dsl_headers::models::{ + HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, ParseFailure, ParseOutcome, +}; +use sqlbuild_rules::dsl_headers::types::HeaderParser; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; +const LINT_TYPES_MODULE: &str = "sqlbuild.lint.types"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const PATHLIB_MODULE: &str = "pathlib"; +const SENTINEL_PATH: &str = ""; +const ENGINE: &str = "sqlbuild"; +const LINE_FEED: &str = "\n"; +const CARRIAGE_RETURN_LINE_FEED: &str = "\r\n"; +const FUNCTION_KIND: &str = "FUNCTION"; + +/// Header values parsed by `parse_header_values`, with Python's error classes. +struct PyHeaderParser<'py> { + parse: Bound<'py, PyAny>, + path: Bound<'py, PyAny>, + syntax_error: Bound<'py, PyAny>, + discovery_error: Bound<'py, PyAny>, + /// The header syntax error of the last rejected parse. + rejected: RefCell>, +} + +impl<'py> PyHeaderParser<'py> { + fn new(py: Python<'py>) -> PyResult { + Ok(Self { + parse: imported(py, NATIVE_MODULE, "parse_header_values")?, + path: imported(py, PATHLIB_MODULE, "Path")?.call1((SENTINEL_PATH,))?, + syntax_error: imported(py, DISCOVERY_ERRORS_MODULE, "ModelSqlParseError")?, + discovery_error: imported(py, DISCOVERY_ERRORS_MODULE, "DiscoveryError")?, + rejected: RefCell::new(None), + }) + } + + fn failure(&self, error: &PyErr) -> PyResult { + let py: Python<'py> = self.parse.py(); + let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); + let help: Option = if value.is_instance(&self.discovery_error)? { + value.getattr("help")?.extract()? + } else { + None + }; + Ok(ParseFailure { + message: value.str()?.extract()?, + help, + }) + } +} + +impl<'py> HeaderParser for PyHeaderParser<'py> { + type Values = Bound<'py, PyAny>; + type Error = PyErr; + + fn parse(&self, kind: &str, inner: &str) -> ParseOutcome { + let py: Python<'py> = self.parse.py(); + let parsed: PyResult> = call_with( + &self.parse, + &[ + ("header", PyString::new(py, inner).into_any()), + ("file_path", self.path.clone()), + ("statement_name", PyString::new(py, kind).into_any()), + ("header_line", py.None().into_bound(py)), + ], + ); + let error: PyErr = match parsed { + Ok(values) => return ParseOutcome::Parsed(values), + Err(error) => error, + }; + let failure: ParseFailure = match self.failure(&error) { + Ok(failure) => failure, + Err(failed) => { + return ParseOutcome::Raised( + failed, + ParseFailure { + message: String::new(), + help: None, + }, + ); + } + }; + if error.is_instance(py, &self.syntax_error) { + self.rejected.replace(Some(error)); + ParseOutcome::Rejected(failure) + } else { + ParseOutcome::Raised(error, failure) + } + } + + fn description(&self, values: &Self::Values) -> PyResult> { + let description: Bound<'py, PyAny> = values.call_method1("get", ("description",))?; + if description.is_instance_of::() { + return description.extract().map(Some); + } + Ok(None) + } + + fn same(&self, left: &Self::Values, right: &Self::Values) -> PyResult { + left.eq(right) + } +} + +/// The models and enums header results are built as. +struct LintClasses<'py> { + violation: Bound<'py, PyAny>, + severity: Bound<'py, PyAny>, +} + +impl<'py> LintClasses<'py> { + fn new(py: Python<'py>) -> PyResult { + Ok(Self { + violation: imported(py, LINT_MODELS_MODULE, "LintViolation")?, + severity: imported(py, LINT_TYPES_MODULE, "LintSeverity")?, + }) + } + + fn violation( + &self, + file_path: &Bound<'py, PyAny>, + violation: HeaderViolation, + ) -> PyResult> { + let py: Python<'py> = file_path.py(); + let severity: &str = match violation.severity { + HeaderSeverity::Fault => "FAULT", + HeaderSeverity::Warning => "WARNING", + }; + call_with( + &self.violation, + &[ + ("file_path", file_path.clone()), + ("line", violation.line.into_pyobject(py)?.into_any()), + ("column", violation.column.into_pyobject(py)?.into_any()), + ("code", PyString::new(py, violation.code).into_any()), + ("message", PyString::new(py, &violation.message).into_any()), + ("severity", self.severity.getattr(severity)?), + ("engine", PyString::new(py, ENGINE).into_any()), + ( + "remediation", + PyString::new(py, &violation.remediation).into_any(), + ), + ], + ) + } + + fn violations( + &self, + file_path: &Bound<'py, PyAny>, + violations: Vec, + ) -> PyResult>> { + violations + .into_iter() + .map(|violation| self.violation(file_path, violation)) + .collect() + } +} + +fn text(contents: &Bound<'_, PyAny>) -> PyResult> { + Ok(contents.extract::()?.chars().collect()) +} + +fn rules_config(config: &Bound<'_, PyAny>) -> PyResult { + Ok(HeaderRulesConfig { + line_width: config.getattr("line_width")?.extract()?, + max_description_lines: config.getattr("max_description_lines")?.extract()?, + }) +} + +fn spans(headers: &Bound<'_, PyAny>) -> PyResult> { + headers + .try_iter()? + .map(|header| { + let header: Bound<'_, PyAny> = header?; + Ok(HeaderSpan { + kind: header.getattr("kind")?.extract()?, + start: header.getattr("start")?.extract()?, + end: header.getattr("end")?.extract()?, + }) + }) + .collect() +} + +fn ranges<'py>(py: Python<'py>, ranges: Vec<(usize, usize)>) -> PyResult> { + PyTuple::new(py, ranges) +} + +/// Every DSL header span in `contents`, or only the first. +#[pyfunction] +#[pyo3(signature = (*, contents, first_only = false))] +fn scan_headers<'py>( + contents: &Bound<'py, PyAny>, + first_only: bool, +) -> PyResult> { + let py: Python<'py> = contents.py(); + let class: Bound<'py, PyAny> = imported(py, LINT_MODELS_MODULE, "HeaderSpan")?; + let headers: Vec> = scan(&text(contents)?, first_only) + .into_iter() + .map(|header| { + call_with( + &class, + &[ + ("kind", PyString::new(py, &header.kind).into_any()), + ("start", header.start.into_pyobject(py)?.into_any()), + ("end", header.end.into_pyobject(py)?.into_any()), + ], + ) + }) + .collect::>()?; + PyTuple::new(py, headers) +} + +/// Non-empty SQL body ranges around and between the headers. +#[pyfunction] +#[pyo3(signature = (*, contents, headers))] +fn sql_body_ranges<'py>( + contents: &Bound<'py, PyAny>, + headers: &Bound<'py, PyAny>, +) -> PyResult> { + ranges( + contents.py(), + body_ranges(&text(contents)?, &spans(headers)?), + ) +} + +/// The authored query ranges inside MEASURE and EVIDENCE blocks. +#[pyfunction] +#[pyo3(signature = (*, contents))] +fn measurement_body_ranges<'py>(contents: &Bound<'py, PyAny>) -> PyResult> { + ranges(contents.py(), measurement_ranges(&text(contents)?)) +} + +fn header_violations<'py>( + contents: &[char], + headers: &[HeaderSpan], + config: &Bound<'py, PyAny>, +) -> PyResult> { + let parser: PyHeaderParser<'py> = PyHeaderParser::new(config.py())?; + lint_headers(contents, headers, rules_config(config)?, &parser) +} + +/// Every header rule violation of `headers` in `contents`. +#[pyfunction] +#[pyo3(signature = (*, contents, file_path, headers, config))] +fn lint_native_headers<'py>( + contents: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + headers: &Bound<'py, PyAny>, + config: &Bound<'py, PyAny>, +) -> PyResult> { + let violations: Vec = + header_violations(&text(contents)?, &spans(headers)?, config)?; + let classes: LintClasses<'py> = LintClasses::new(contents.py())?; + PyTuple::new(contents.py(), classes.violations(file_path, violations)?) +} + +/// `contents` with header formatting fixes applied, and the faults relocation reported. +#[pyfunction] +#[pyo3(signature = (*, contents, file_path, config))] +fn format_native_headers<'py>( + contents: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + config: &Bound<'py, PyAny>, +) -> PyResult<(String, Bound<'py, PyTuple>)> { + let py: Python<'py> = contents.py(); + let parser: PyHeaderParser<'py> = PyHeaderParser::new(py)?; + let (formatted, faults) = format_headers(&text(contents)?, rules_config(config)?, &parser)?; + let classes: LintClasses<'py> = LintClasses::new(py)?; + Ok(( + formatted.into_iter().collect(), + PyTuple::new(py, classes.violations(file_path, faults)?)?, + )) +} + +/// The header-parse faults of one file. +fn parse_faults<'py>( + classes: &LintClasses<'py>, + (file_path, contents): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + config: &Bound<'py, PyAny>, +) -> PyResult>> { + let contents: Vec = text(contents)?; + let violations: Vec = + header_violations(&contents, &scan(&contents, false), config)? + .into_iter() + .filter(|violation| violation.code == RULE_HEADER_PARSE) + .collect(); + classes.violations(file_path, violations) +} + +/// Batch parse formatter-owned headers before per-file formatting and linting. +#[pyfunction] +#[pyo3(signature = (*, files))] +fn prepare_native_header_cache(files: &Bound<'_, PyDict>) -> PyResult<()> { + let mut headers: Vec = Vec::new(); + for contents in files.values() { + let contents: Vec = text(&contents)?; + for header in scan(&contents, false) { + headers.push( + inner_header_text(&contents[header.start..header.end]) + .iter() + .collect(), + ); + } + } + imported(files.py(), NATIVE_MODULE, "prepare_model_header_tokens")?.call1((headers,))?; + Ok(()) +} + +/// Parse-safe files, and every file's authored newline convention. +#[pyfunction] +#[pyo3(signature = (*, files, config))] +fn safe_format_files<'py>( + files: &Bound<'py, PyDict>, + config: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { + let py: Python<'py> = files.py(); + let classes: LintClasses<'py> = LintClasses::new(py)?; + let safe: Bound<'py, PyDict> = PyDict::new(py); + let newlines: Bound<'py, PyDict> = PyDict::new(py); + for (file_path, contents) in files.iter() { + if parse_faults(&classes, (&file_path, &contents), config)?.is_empty() { + safe.set_item(&file_path, &contents)?; + } + } + for (file_path, contents) in files.iter() { + let crlf: bool = contents + .extract::()? + .contains(CARRIAGE_RETURN_LINE_FEED); + newlines.set_item( + file_path, + if crlf { + CARRIAGE_RETURN_LINE_FEED + } else { + LINE_FEED + }, + )?; + } + Ok((safe, newlines)) +} + +/// Every rewrite whose header still parses, with the faults of the discarded ones appended. +#[pyfunction] +#[pyo3(signature = (*, updated, config, faults))] +fn reject_unparseable_header_rewrites<'py>( + updated: &Bound<'py, PyDict>, + config: &Bound<'py, PyAny>, + faults: &Bound<'py, PyList>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyList>)> { + let classes: LintClasses<'py> = LintClasses::new(updated.py())?; + let accepted: Bound<'py, PyDict> = updated.copy()?; + for (file_path, contents) in updated.iter() { + let parse: Vec> = + parse_faults(&classes, (&file_path, &contents), config)?; + if parse.is_empty() { + continue; + } + if accepted.contains(&file_path)? { + accepted.del_item(&file_path)?; + } + for fault in parse { + faults.append(fault)?; + } + } + Ok((accepted, faults.clone())) +} + +/// The SQL function argument names a FUNCTION header declares, lowercased and sorted. +#[pyfunction] +#[pyo3(signature = (*, contents, headers))] +fn external_identifiers_for_headers<'py>( + contents: &Bound<'py, PyAny>, + headers: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = contents.py(); + let text: Vec = text(contents)?; + let parser: PyHeaderParser<'py> = PyHeaderParser::new(py)?; + for header in spans(headers)? { + if header.kind != FUNCTION_KIND { + continue; + } + let inner: String = inner_header_text(&text[header.start..header.end]) + .iter() + .collect(); + let values: Bound<'py, PyAny> = match parser.parse(&header.kind, &inner) { + ParseOutcome::Parsed(values) => values, + ParseOutcome::Rejected(_) => { + return Err(parser.rejected.take().unwrap_or_else(|| PyErr::fetch(py))); + } + ParseOutcome::Raised(error, _) => return Err(error), + }; + let arguments: Bound<'py, PyAny> = values.call_method1("get", ("arguments",))?; + let Ok(arguments) = arguments.downcast::() else { + continue; + }; + let mut names: Vec = arguments + .keys() + .iter() + .map(|name| name.str()?.call_method0("lower")?.extract()) + .collect::>()?; + names.sort_unstable(); + return PyTuple::new(py, names); + } + Ok(PyTuple::empty(py)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scan_headers, module)?)?; + module.add_function(wrap_pyfunction!(sql_body_ranges, module)?)?; + module.add_function(wrap_pyfunction!(measurement_body_ranges, module)?)?; + module.add_function(wrap_pyfunction!(lint_native_headers, module)?)?; + module.add_function(wrap_pyfunction!(format_native_headers, module)?)?; + module.add_function(wrap_pyfunction!(prepare_native_header_cache, module)?)?; + module.add_function(wrap_pyfunction!(safe_format_files, module)?)?; + module.add_function(wrap_pyfunction!( + reject_unparseable_header_rewrites, + module + )?)?; + module.add_function(wrap_pyfunction!(external_identifiers_for_headers, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index c10cf564a2..9956d91032 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -1,5 +1,9 @@ +pub(crate) mod custom_rule_evidence; +pub(crate) mod dsl_headers; pub(crate) mod evaluation; pub(crate) mod lint; +pub(crate) mod registration; pub(crate) mod relation_keys; +pub(crate) mod results; pub(crate) mod rows; pub(crate) mod skills; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs new file mode 100644 index 0000000000..a9de0613e5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -0,0 +1,16 @@ +//! Register the native rules, lint and DSL header functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::rules::{ + custom_rule_evidence, dsl_headers, evaluation, lint, results, skills, +}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + evaluation::register(module)?; + lint::register(module)?; + dsl_headers::register(module)?; + custom_rule_evidence::register(module)?; + results::register(module)?; + skills::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs new file mode 100644 index 0000000000..5a0bcbd5ec --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs @@ -0,0 +1,106 @@ +//! Rules results rendered for the CLI, and SQBRSQL044 help naming files that repeat a literal. + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::rule_results::main::duplicate_literal_hints::duplicate_literal_hints; +use sqlbuild_rules::rule_results::main::format_result::format_result; +use sqlbuild_rules::rule_results::models::{ResultCounts, RuleFinding}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const DATACLASSES_MODULE: &str = "dataclasses"; +const UTF8: &str = "utf-8"; + +fn rule_finding(finding: &Bound<'_, PyAny>) -> PyResult { + Ok(RuleFinding { + code: finding.getattr("code")?.extract()?, + path: finding + .getattr("path")? + .call_method0("as_posix")? + .extract()?, + line: finding.getattr("line")?.extract()?, + column: finding.getattr("column")?.extract()?, + message: finding.getattr("message")?.extract()?, + remediation: finding.getattr("remediation")?.extract()?, + unevaluated: finding.getattr("unevaluated")?.is_truthy()?, + affected_rules: finding.getattr("affected_rules")?.extract()?, + fixable: finding.getattr("fixable")?.is_truthy()?, + }) +} + +fn rule_findings(findings: &Bound<'_, PyAny>) -> PyResult> { + findings + .try_iter()? + .map(|finding| rule_finding(&finding?)) + .collect() +} + +/// One rules result for the requested CLI surface. +#[pyfunction] +#[pyo3(signature = (*, result, json_output))] +fn format_rules_result(result: &Bound<'_, PyAny>, json_output: bool) -> PyResult { + let counts = ResultCounts { + evaluated_models: result.getattr("evaluated_models")?.extract()?, + unevaluated_resources: result.getattr("unevaluated_resources")?.extract()?, + cache_hits: result.getattr("cache_hits")?.extract()?, + cache_misses: result.getattr("cache_misses")?.extract()?, + }; + Ok(format_result( + &rule_findings(&result.getattr("findings")?)?, + counts, + json_output, + )) +} + +/// The authored lines of one finding's file; an unreadable file has none. +fn file_lines(project_dir: &Bound<'_, PyAny>, path: &str) -> PyResult> { + let py = project_dir.py(); + let file: Bound<'_, PyAny> = project_dir.div(path)?; + let read: PyResult> = call_with( + &file.getattr("read_text")?, + &[("encoding", PyString::new(py, UTF8).into_any())], + ); + match read { + Ok(text) => text.call_method0("splitlines")?.extract(), + Err(error) if error.is_instance_of::(py) => Ok(Vec::new()), + Err(error) => Err(error), + } +} + +/// Prefix SQBRSQL044 help with the other files that repeat the identical literal. +#[pyfunction] +#[pyo3(signature = (*, findings, project_dir))] +fn with_duplicate_literal_hints<'py>( + findings: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py = findings.py(); + let originals: Vec> = findings.try_iter()?.collect::>()?; + let rows: Vec = originals + .iter() + .map(rule_finding) + .collect::>()?; + let hinted: Vec> = + duplicate_literal_hints::(&rows, |path| file_lines(project_dir, path))?; + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let mut output: Vec> = Vec::with_capacity(originals.len()); + for (finding, remediation) in originals.into_iter().zip(hinted) { + output.push(match remediation { + Some(remediation) => { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("remediation", remediation)?; + replace.call((finding,), Some(&keywords))? + } + None => finding, + }); + } + PyTuple::new(py, output) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_rules_result, module)?)?; + module.add_function(wrap_pyfunction!(with_duplicate_literal_hints, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs index 047e1c0225..d5d93cd1f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod registration; pub(crate) mod relationship_names; pub(crate) mod relationship_scan; pub(crate) mod scope_cache; +pub(crate) mod scope_command; pub(crate) mod scope_errors; pub(crate) mod scope_index; pub(crate) mod scope_loading; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs index 8273ccd011..59556826ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs @@ -3,8 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::scopes::{ - compile_usages, declaration_contexts, relationship_names, scope_cache, scope_index, - scope_loading, scope_lookup, scope_reports, scope_validation, scope_visibility, + compile_usages, declaration_contexts, relationship_names, scope_cache, scope_command, + scope_index, scope_loading, scope_lookup, scope_reports, scope_validation, scope_visibility, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -16,6 +16,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { scope_cache::register(module)?; scope_loading::register(module)?; scope_reports::register(module)?; + scope_command::register(module)?; scope_validation::register(module)?; scope_visibility::register(module)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs new file mode 100644 index 0000000000..bfef737d93 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -0,0 +1,364 @@ +//! `sqb scope`: load offline compiler facts, run one scope query and present its result. + +use std::collections::BTreeSet; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_scopes::scope_report::models::ScopeResult; +use sqlbuild_scopes::scope_text::main::render_scope_text::render_scope_text; +use sqlbuild_scopes::scope_text::main::scope_result_failed::scope_result_failed; +use sqlbuild_scopes::scope_text::models::ScopeTextRequest; + +use crate::bindings::_helpers::scopes::scope_objects::ScopeClasses; +use crate::bindings::_helpers::scopes::scope_report_objects::read_result; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const TARGET_ERROR_CODE: &str = "C960"; +const BARE_USAGE: &str = "bare names work for models, sources, seeds and functions; write macros, \ +enums, constants, tests and audits with their prefix, for example macro:"; +/// Resource kinds a bare target name may refer to. +const BARE_KINDS: [&str; 4] = ["model", "source", "seed", "function"]; +const PATH_MARKERS: [char; 3] = ['/', '\\', '.']; + +fn keywords<'py>( + py: Python<'py>, + items: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in items { + dict.set_item(*name, value)?; + } + Ok(dict) +} + +fn user_error(py: Python<'_>, message: String) -> PyResult { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", TARGET_ERROR_CODE)?; + options.set_item("help", BARE_USAGE)?; + let error: Bound<'_, PyAny> = py + .import(ERRORS_MODULE)? + .getattr("CliUserError")? + .call((message,), Some(&options))?; + Ok(PyErr::from_value(error)) +} + +fn or_list(values: &[String]) -> String { + match values.split_last() { + Some((last, rest)) if !rest.is_empty() => format!("{} or {last}", rest.join(", ")), + _ => values.join(""), + } +} + +/// The kinds of resources named `target`, and the prefixed names of other matches. +fn target_matches( + lookup: &Bound<'_, PyAny>, + target: &str, +) -> PyResult<(BTreeSet, Vec)> { + let mut bare: BTreeSet = BTreeSet::new(); + let mut prefixed: BTreeSet = BTreeSet::new(); + for identity in lookup.getattr("resources")?.try_iter()? { + let identity: Bound<'_, PyAny> = identity?; + if identity.getattr("name")?.extract::()? != target { + continue; + } + let kind: String = identity.getattr("kind")?.str()?.extract()?; + if BARE_KINDS.contains(&kind.as_str()) { + bare.insert(kind); + } else { + prefixed.insert(format!("{kind}:{target}")); + } + } + for identity in lookup.getattr("declarations")?.try_iter()? { + let identity: Bound<'_, PyAny> = identity?; + if identity.getattr("owner")?.is_none() + && identity.getattr("name")?.extract::()? == target + { + let kind: String = identity.getattr("kind")?.str()?.extract()?; + prefixed.insert(format!("{kind}:{target}")); + } + } + Ok((bare, prefixed.into_iter().collect())) +} + +/// `target` with its kind prefix when it is a bare model, source, seed or function. +fn qualify_target( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + target: Option, +) -> PyResult> { + let Some(target) = target else { + return Ok(None); + }; + if target.contains(':') || target.contains(PATH_MARKERS) { + return Ok(Some(target)); + } + let (bare, prefixed) = target_matches(lookup, &target)?; + if bare.is_empty() { + let message: String = if prefixed.is_empty() { + format!("unknown scope target '{target}'") + } else { + format!( + "scope target '{target}' needs its prefix: {}", + or_list(&prefixed) + ) + }; + return Err(user_error(py, message)?); + } + let candidates: Vec = bare + .iter() + .map(|kind| format!("{kind}:{target}")) + .chain(prefixed) + .collect(); + if candidates.len() > 1 { + return Err(user_error( + py, + format!( + "scope target '{target}' matches more than one resource or declaration; write {}", + or_list(&candidates) + ), + )?); + } + Ok(candidates.into_iter().next()) +} + +fn text_request(request: &Bound<'_, PyAny>, target: Option) -> PyResult { + let mut kinds: Vec = Vec::new(); + for kind in request.getattr("kinds")?.try_iter()? { + kinds.push(kind?.extract()?); + } + Ok(ScopeTextRequest { + target, + at: request.getattr("at")?.extract()?, + as_path: request.getattr("as_path")?.extract()?, + defined_under: request.getattr("defined_under")?.extract()?, + kinds, + glob: request.getattr("match")?.extract()?, + used_only: request.getattr("used_only")?.extract()?, + include_nearby: request.getattr("include_nearby")?.extract()?, + nearby_depth: request.getattr("nearby_depth")?.extract()?, + dependency_depth: request.getattr("dependency_depth")?.extract()?, + explain: request.getattr("explain")?.extract()?, + globals: request.getattr("globals")?.extract()?, + page_size: request.getattr("page_size")?.extract()?, + paths: request.getattr("paths")?.extract()?, + verbose: request.getattr("verbose")?.extract()?, + no_cache: request.getattr("no_cache")?.extract()?, + }) +} + +fn report_filters<'py>( + py: Python<'py>, + request: &Bound<'py, PyAny>, +) -> PyResult> { + let kind_class: Bound<'_, PyAny> = py.import(TYPES_MODULE)?.getattr("DeclarationKind")?; + let mut kinds: Vec> = Vec::new(); + for kind in request.getattr("kinds")?.try_iter()? { + kinds.push(kind_class.call1((kind?,))?); + } + let options: Bound<'_, PyDict> = keywords( + py, + &[ + ("include_nearby", request.getattr("include_nearby")?), + ("defined_under", request.getattr("defined_under")?), + ("kinds", PyTuple::new(py, kinds)?.into_any()), + ("glob", request.getattr("match")?), + ("used_only", request.getattr("used_only")?), + ("dependency_depth", request.getattr("dependency_depth")?), + ("cursor", request.getattr("after")?), + ("page_size", request.getattr("page_size")?), + ("nearby_depth", request.getattr("nearby_depth")?), + ("globals", request.getattr("globals")?), + ], + )?; + py.import(MODELS_MODULE)? + .getattr("ScopeReportFilters")? + .call((), Some(&options)) +} + +/// `dataclasses.replace(report, **changes)`. +fn replaced<'py>( + report: &Bound<'py, PyAny>, + changes: &Bound<'py, PyDict>, +) -> PyResult> { + report + .py() + .import("dataclasses")? + .getattr("replace")? + .call((report,), Some(changes)) +} + +/// The report with extra diagnostics and a part of the result merged in. +fn merged<'py>( + report: &Bound<'py, PyAny>, + (name, value): (&str, Bound<'py, PyAny>), + (diagnostics, complete): (Bound<'py, PyAny>, bool), +) -> PyResult> { + let py: Python<'py> = report.py(); + let combined: Bound<'py, PyAny> = report.getattr("diagnostics")?.add(diagnostics)?; + let complete: bool = report.getattr("complete")?.extract::()? && complete; + let changes: Bound<'py, PyDict> = keywords( + py, + &[ + (name, value), + ("diagnostics", combined), + ( + "complete", + pyo3::types::PyBool::new(py, complete).to_owned().into_any(), + ), + ], + )?; + replaced(report, &changes) +} + +/// Run the query `request` selects over `lookup`. +fn query_result<'py>( + py: Python<'py>, + lookup: &Bound<'py, PyAny>, + (request, target): (&Bound<'py, PyAny>, Option), +) -> PyResult> { + let native: Bound<'_, PyModule> = py.import(NATIVE_MODULE)?; + let at: Option = request.getattr("at")?.extract()?; + let prospective: bool = at.is_some(); + let browse: Option = request.getattr("browse")?.extract()?; + if let Some(folder) = browse { + return native + .getattr("scope_browse")? + .call1((lookup, (folder, &target, prospective))); + } + let filters: Bound<'_, PyAny> = report_filters(py, request)?; + let list_path: Option = request.getattr("list_path")?.extract()?; + if let Some(folder) = list_path { + return native + .getattr("scope_list")? + .call1((lookup, (folder, &filters, &target, prospective))); + } + let directory: bool = at + .as_deref() + .is_some_and(|at| at.ends_with('/') || at.ends_with('\\')); + let mut report: Bound<'_, PyAny> = native + .getattr("scope_report")? + .call1((lookup, (&target, &at, directory, &filters)))?; + let explain: Option = request.getattr("explain")?.extract()?; + if let Some(declaration) = explain { + let (explanation, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = native + .getattr("scope_explain")? + .call1((lookup, (declaration, &target, &at, directory)))? + .extract()?; + let complete: bool = explanation.getattr("complete")?.extract()?; + report = merged( + &report, + ("explanation", explanation), + (diagnostics, complete), + )?; + } + let as_path: Option = request.getattr("as_path")?.extract()?; + if let Some(destination) = as_path { + let (preview, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = native + .getattr("scope_move_preview")? + .call1((lookup, target.clone().unwrap_or_default(), destination))? + .extract()?; + let complete: bool = !preview.is_none() && preview.getattr("complete")?.extract()?; + report = merged(&report, ("move_preview", preview), (diagnostics, complete))?; + } + Ok(report) +} + +fn use_color( + py: Python<'_>, + request: &Bound<'_, PyAny>, + output_stream: Option<&Bound<'_, PyAny>>, +) -> PyResult { + if output_stream.is_some() || request.getattr("no_color")?.extract::()? { + return Ok(false); + } + py.import(COLOR_MODULE)? + .getattr("supports_color")? + .call0()? + .extract() +} + +/// Load offline compiler facts, run one pure query, and present its result. +#[pyfunction] +#[pyo3(signature = (*, request, load_scope_index = None, output_stream = None))] +fn run_scope_command( + py: Python<'_>, + request: &Bound<'_, PyAny>, + load_scope_index: Option>, + output_stream: Option>, +) -> PyResult { + let native: Bound<'_, PyModule> = py.import(NATIVE_MODULE)?; + let project_dir: Bound<'_, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, + }; + let loader: Bound<'_, PyAny> = match load_scope_index { + Some(loader) => loader, + None => native.getattr("load_or_build_scope_index")?, + }; + let options: Bound<'_, PyDict> = keywords( + py, + &[ + ("project_dir", project_dir), + ("no_cache", request.getattr("no_cache")?), + ], + )?; + let index: Bound<'_, PyAny> = loader.call((), Some(&options))?; + let lookup: Bound<'_, PyAny> = native.getattr("build_scope_lookup")?.call1((index,))?; + let target: Option = + qualify_target(py, &lookup, request.getattr("target")?.extract()?)?; + let result: Bound<'_, PyAny> = query_result(py, &lookup, (request, target.clone()))?; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let (output, native_result): (String, ScopeResult) = + if request.getattr("json_output")?.extract()? { + let output: String = native + .getattr("serialize_scope_report")? + .call1((&result,))? + .extract()?; + (output, read_result(&classes, &result)?) + } else { + let native_result: ScopeResult = read_result(&classes, &result)?; + let style: CliStyle = CliStyle { + use_color: use_color(py, request, output_stream.as_ref())?, + }; + let output: String = + render_scope_text(&native_result, &text_request(request, target)?, style); + (output, native_result) + }; + let stream: Bound<'_, PyAny> = match output_stream { + Some(stream) => stream, + None => py.import("sys")?.getattr("stdout")?, + }; + stream.call_method1("write", (output,))?; + Ok(i32::from(scope_result_failed(&native_result))) +} + +/// Render one compiler-owned scope result as `sqb scope` text. +#[pyfunction] +#[pyo3(signature = (*, result, request, use_color = false))] +fn render_scope_result( + py: Python<'_>, + result: &Bound<'_, PyAny>, + request: &Bound<'_, PyAny>, + use_color: bool, +) -> PyResult { + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let target: Option = request.getattr("target")?.extract()?; + Ok(render_scope_text( + &read_result(&classes, result)?, + &text_request(request, target)?, + CliStyle { use_color }, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_scope_command, module)?)?; + module.add_function(wrap_pyfunction!(render_scope_result, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs new file mode 100644 index 0000000000..28fd26df81 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs @@ -0,0 +1,217 @@ +//! `sqb playground`, `sqb skills` and generated-skill maintenance as `_native` functions. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use pyo3::exceptions::PyUnicodeDecodeError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::workspace::errors::WorkspaceError; +use sqlbuild_config::workspace::main::create_playground::create_playground; +use sqlbuild_config::workspace::main::install_skills::install_skills; +use sqlbuild_config::workspace::main::maintain_skills::maintain_skills; +use sqlbuild_config::workspace::main::packaged_skill_files::packaged_skill_files; +use sqlbuild_config::workspace::main::playground_project_dir::playground_project_dir; +use sqlbuild_config::workspace::main::playground_summary::playground_summary; +use sqlbuild_config::workspace::models::{PlaygroundRequest, SkillInstallRequest}; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; + +const SKILLS_DIRECTORY: &str = ".agents/skills/sqlbuild"; +const TEMPLATES_DIRECTORY: &str = "playground_templates"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const OUTPUT_MODELS: &str = "sqlbuild.cli.output.models"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const SKILLS_OPTIONS: (&[&str], &[&str]) = + (&["project_dir"], &["global_install", "targets", "force"]); +const UPDATE_OPTIONS: (&[&str], &[&str]) = ( + &["project_dir"], + &["global_install", "requested_targets", "force", "home_dir"], +); + +/// The Python exception the workspace failure is raised as. +fn raised(py: Python<'_>, error: WorkspaceError) -> PyErr { + match error { + WorkspaceError::User { + code, + message, + help, + } => { + let created = || -> PyResult { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", code)?; + keywords.set_item("help", help)?; + let instance: Bound<'_, PyAny> = imported(py, EXCEPTIONS_MODULE, "CliUserError")? + .call((message,), Some(&keywords))?; + Ok(PyErr::from_value(instance)) + }; + created().unwrap_or_else(|failure| failure) + } + WorkspaceError::Decode { bytes, error, .. } => { + match PyUnicodeDecodeError::new_utf8(py, &bytes, error) { + Ok(instance) => PyErr::from_value(instance.into_any()), + Err(failure) => failure, + } + } + WorkspaceError::Io(error) => PyErr::from(error), + } +} + +/// The installed `sqlbuild` package directory. +fn package_dir(py: Python<'_>) -> PyResult { + let file: PathBuf = py.import("sqlbuild")?.getattr("__file__")?.extract()?; + Ok(file.parent().map(Path::to_path_buf).unwrap_or_default()) +} + +fn home_dir(py: Python<'_>) -> PyResult { + imported(py, "pathlib", "Path")? + .call_method0("home")? + .extract() +} + +fn current_dir(py: Python<'_>) -> PyResult { + imported(py, "pathlib", "Path")? + .call_method0("cwd")? + .extract() +} + +fn install(py: Python<'_>, request: &SkillInstallRequest) -> PyResult> { + install_skills(request).map_err(|error| raised(py, error)) +} + +fn skill_request(py: Python<'_>, project_dir: PathBuf) -> PyResult { + Ok(SkillInstallRequest { + project_dir, + skills_root: package_dir(py)?.join(SKILLS_DIRECTORY), + home_dir: home_dir(py)?, + global_install: false, + requested_targets: Vec::new(), + force: false, + }) +} + +/// Write packaged SQLBuild skill content to configured agent locations. +#[pyfunction] +#[pyo3(signature = (**options))] +fn update_sqlbuild_skills<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "update_sqlbuild_skills", options, UPDATE_OPTIONS)?; + let mut request: SkillInstallRequest = + skill_request(py, arguments.value("project_dir")?.extract()?)?; + request.global_install = arguments.value_or_false("global_install")?.is_truthy()?; + request.force = arguments.value_or_false("force")?.is_truthy()?; + if arguments.passed("requested_targets")? { + request.requested_targets = arguments.value("requested_targets")?.extract()?; + } + let home: Bound<'py, PyAny> = arguments.value("home_dir")?; + if !home.is_none() { + request.home_dir = home.extract()?; + } + let written: Vec = install(py, &request)?; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("written_paths", PyTuple::new(py, written)?)?; + imported(py, OUTPUT_MODELS, "SkillUpdateResult")?.call((), Some(&keywords)) +} + +/// Check or refresh generated project skills without changing command success. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +fn maintain_sqlbuild_skills(py: Python<'_>, project_dir: PathBuf) -> PyResult> { + let message: String = maintain_skills(&project_dir, &package_dir(py)?.join(SKILLS_DIRECTORY)) + .map_err(|error| raised(py, error))?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("message", message)?; + imported(py, OUTPUT_MODELS, "SkillMaintenanceResult")?.call((), Some(&keywords)) +} + +/// Return every packaged skill file keyed by skill-relative path, with generated markers. +#[pyfunction] +fn load_packaged_skill_files(py: Python<'_>) -> PyResult> { + packaged_skill_files(&package_dir(py)?.join(SKILLS_DIRECTORY)) + .map_err(|error| raised(py, error)) +} + +fn print(py: Python<'_>, text: &str) -> PyResult<()> { + let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + stdout.call_method1("write", (text,))?; + Ok(()) +} + +/// `sqb skills`: install or update SQLBuild skill files. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_skills_command(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "run_skills_command", options, SKILLS_OPTIONS)?; + let project_dir: Bound<'_, PyAny> = arguments.value("project_dir")?; + let base: PathBuf = if project_dir.is_none() { + current_dir(py)? + } else { + project_dir.extract()? + }; + let mut request: SkillInstallRequest = skill_request(py, base)?; + request.global_install = arguments.value_or_false("global_install")?.is_truthy()?; + request.force = arguments.value_or_false("force")?.is_truthy()?; + if arguments.passed("targets")? { + request.requested_targets = arguments.value("targets")?.extract()?; + } + let written: Vec = install(py, &request)?; + let mut text: String = "Updated SQLBuild skill files:\n".to_owned(); + for path in written { + text.push_str(&format!(" {}\n", path.display())); + } + print(py, &text)?; + Ok(0) +} + +/// `sqb playground`: create a self-contained playground project. +#[pyfunction] +fn run_playground_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let project_dir: Bound<'_, PyAny> = request.getattr("project_dir")?; + let base: PathBuf = if project_dir.is_none() { + current_dir(py)? + } else { + project_dir.extract()? + }; + let display_path: String = request.getattr("target_path")?.str()?.extract()?; + let target_path: PathBuf = PathBuf::from(&display_path); + let target_dir: PathBuf = if target_path.is_absolute() { + target_path + } else { + base.join(target_path) + }; + let template: String = request.getattr("template")?.str()?.extract()?; + create_playground(&PlaygroundRequest { + templates_root: package_dir(py)?.join(TEMPLATES_DIRECTORY), + target_dir: target_dir.clone(), + template: template.clone(), + }) + .map_err(|error| raised(py, error))?; + install( + py, + &skill_request(py, playground_project_dir(&target_dir, &template))?, + )?; + let use_color: bool = imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?; + print( + py, + &playground_summary(&display_path, &template, CliStyle { use_color }), + )?; + Ok(0) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(update_sqlbuild_skills, module)?)?; + module.add_function(wrap_pyfunction!(maintain_sqlbuild_skills, module)?)?; + module.add_function(wrap_pyfunction!(load_packaged_skill_files, module)?)?; + module.add_function(wrap_pyfunction!(run_skills_command, module)?)?; + module.add_function(wrap_pyfunction!(run_playground_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/mod.rs new file mode 100644 index 0000000000..0b3c75ebc0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod commands; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/registration.rs new file mode 100644 index 0000000000..310164278b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/registration.rs @@ -0,0 +1,9 @@ +//! Register the playground and skills commands. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::workspace_commands::commands; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + commands::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 0d651d4ab2..70d923473f 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,19 +5,18 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, - compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, - contracts, dag_artifact, discovery, fingerprints, frontier, graph, lineage, macro_bridge, - manifest, model_config, plan_output, project_assembly, refactor, references, render, + command_runtime, compile_attachment, compile_command, compile_driver, compile_outputs, + compiled_project, contracts, cost_report, cursor, dag_artifact, debug_command, discovery, + fingerprints, freshness, frontier, graph, janitor, lineage, macro_bridge, manifest, + model_config, plan_output, presentation_text, project_assembly, refactor, references, render, result_output, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, - type_system, + type_system, workspace_commands, }; use crate::bindings::models; #[pymodule] pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { - rules::evaluation::register(module)?; - rules::lint::register(module)?; - rules::skills::register(module)?; + rules::registration::register(module)?; cli::registration::register(module)?; module.add( "NativeCompilerError", @@ -46,11 +45,19 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { cache::registration::register(module)?; attachments::registration::register(module)?; refactor::registration::register(module)?; + janitor::registration::register(module)?; + freshness::registration::register(module)?; plan_output::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; result_output::registration::register(module)?; + cost_report::registration::register(module)?; + debug_command::registration::register(module)?; + command_runtime::registration::register(module)?; + presentation_text::registration::register(module)?; + workspace_commands::registration::register(module)?; auditing::registration::register(module)?; + cursor::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-refactor/src/lib.rs b/crates/sqlbuild-refactor/src/lib.rs index 449aab4e2f..4353068a6e 100644 --- a/crates/sqlbuild-refactor/src/lib.rs +++ b/crates/sqlbuild-refactor/src/lib.rs @@ -2,4 +2,5 @@ #![forbid(unsafe_code)] +pub mod refactor_command; pub mod refactoring; diff --git a/crates/sqlbuild-refactor/src/refactor_command/_helpers/json.rs b/crates/sqlbuild-refactor/src/refactor_command/_helpers/json.rs new file mode 100644 index 0000000000..f55f510774 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/_helpers/json.rs @@ -0,0 +1,81 @@ +//! JSON values of refactoring plans. + +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::refactor_command::_helpers::output_text::sorted_edits; +use crate::refactoring::models::{FileChange, ManualLocation, RefactorOperation, RefactorPlan}; + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +pub(crate) fn optional_text(value: Option<&str>) -> JsonValue { + value.map_or(JsonValue::Null, text) +} + +pub(crate) fn integer(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, |value| { + JsonValue::Integer(JsonInteger::from(value)) + }) +} + +pub(crate) fn count(value: usize) -> JsonValue { + JsonValue::Integer(JsonInteger::from(u64::try_from(value).unwrap_or(u64::MAX))) +} + +fn optional_count(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, count) +} + +pub(crate) fn object(entries: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + entries + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ) +} + +pub(crate) fn file(change: &FileChange) -> JsonValue { + object(vec![ + ("path", text(&change.path)), + ( + "from_path", + optional_text(change.moved().then_some(change.original_path.as_str())), + ), + ( + "edits", + JsonValue::Array( + sorted_edits(change) + .into_iter() + .map(|edit| { + object(vec![ + ("line", count(edit.line)), + ("column", count(edit.column)), + ("kind", text(edit.kind.value())), + ("before", text(&edit.before)), + ("after", text(&edit.after)), + ]) + }) + .collect(), + ), + ), + ]) +} + +pub(crate) fn location(item: &ManualLocation) -> JsonValue { + object(vec![ + ("path", text(&item.path)), + ("line", optional_count(item.line)), + ("column", optional_count(item.column)), + ("reason", text(&item.reason)), + ]) +} + +pub(crate) fn operation_value(plan: &RefactorPlan) -> &'static str { + match plan.request.operation { + RefactorOperation::RenameModel => "rename_model", + RefactorOperation::MoveModel => "move_model", + RefactorOperation::RenameColumn => "rename_column", + } +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/_helpers/layer_folders.rs b/crates/sqlbuild-refactor/src/refactor_command/_helpers/layer_folders.rs new file mode 100644 index 0000000000..24fe84f8ad --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/_helpers/layer_folders.rs @@ -0,0 +1,74 @@ +//! Layer folders read from model names and paths, as the layer-folder rules read them. + +use crate::refactor_command::constants::{ + CONTIGUOUS_LAYER_FOLDERS, CURRENT_DIRECTORY, LAYER_FOLDERS, MODEL_NAME_LAYER_INDEX, + MODEL_NAME_LAYER_SEPARATOR, MODEL_NAME_PART_COUNTS, +}; + +pub(crate) fn folder_of( + folders: &[(&str, &'static [&'static str])], + layer: &str, +) -> Option<&'static [&'static str]> { + folders + .iter() + .find(|(name, _)| *name == layer) + .map(|(_, folder)| *folder) +} + +/// The layer segment of a name in the rule grammar, as the layer-folder rules read it. +pub(crate) fn name_layer(name: &str) -> Option<&str> { + let parts: Vec<&str> = name.split(MODEL_NAME_LAYER_SEPARATOR).collect(); + if MODEL_NAME_PART_COUNTS.contains(&parts.len()) { + return Some(parts[MODEL_NAME_LAYER_INDEX]); + } + None +} + +/// `PurePosixPath(path).parts`. +pub(crate) fn path_parts(path: &str) -> Vec<&str> { + let mut parts: Vec<&str> = Vec::new(); + if path.starts_with('/') { + parts.push("/"); + } + parts.extend( + path.split('/') + .filter(|part| !part.is_empty() && *part != CURRENT_DIRECTORY), + ); + parts +} + +/// Swap the current layer's own folder for the new one, keeping the folders around it. +pub(crate) fn mirrored_folder( + original_path: &str, + current_layer: Option<&str>, + layer_folder: &[&str], +) -> Option<(Vec, Vec)> { + let current_layer: &str = current_layer.unwrap_or_default(); + let candidates: Vec<&[&str]> = [ + folder_of(&LAYER_FOLDERS, current_layer), + folder_of(&CONTIGUOUS_LAYER_FOLDERS, current_layer), + ] + .into_iter() + .flatten() + .collect(); + let mut parent: Vec<&str> = path_parts(original_path); + parent.pop(); + for current in candidates { + for start in 1..parent.len() { + let end: usize = (start + current.len()).min(parent.len()); + if parent[start..end] == *current { + let head: Vec = parent[..start] + .iter() + .chain(layer_folder) + .map(|part| (*part).to_owned()) + .collect(); + let tail: Vec = parent[end..] + .iter() + .map(|part| (*part).to_owned()) + .collect(); + return Some((head, tail)); + } + } + } + None +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/_helpers/mod.rs b/crates/sqlbuild-refactor/src/refactor_command/_helpers/mod.rs new file mode 100644 index 0000000000..51d2f42cf2 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod json; +pub(crate) mod layer_folders; +pub(crate) mod output_text; +pub(crate) mod quoting; +pub(crate) mod text; diff --git a/crates/sqlbuild-refactor/src/refactor_command/_helpers/output_text.rs b/crates/sqlbuild-refactor/src/refactor_command/_helpers/output_text.rs new file mode 100644 index 0000000000..2af9c4ee12 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/_helpers/output_text.rs @@ -0,0 +1,205 @@ +//! Lines of the human refactoring output. + +use sqlbuild_core::cli_text::main::format_count_noun::format_count_noun; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::refactor_command::_helpers::text::{ + excerpt, first_line, left_justified, tree_connector, +}; +use crate::refactor_command::constants::{ + BLANK_PREFIX, CONTINUE_PREFIX, FOLDER_PLACEHOLDER, KIND_WIDTH, POSITION_WIDTH, +}; +use crate::refactor_command::models::{LayerMove, RefactorDiagnostic, RefactorStatus}; +use crate::refactoring::models::{ + FileChange, ManualLocation, MigrationDeclaration, RefactorOperation, RefactorPlan, TextEdit, +}; + +/// `model:` or `column:.`. +pub(crate) fn refactor_target(plan: &RefactorPlan) -> String { + let request = &plan.request; + if request.operation == RefactorOperation::RenameColumn { + return format!( + "column:{}.{}", + request.model_name, + request.column_name.as_deref().unwrap_or("None") + ); + } + format!("model:{}", request.model_name) +} + +pub(crate) fn heading(plan: &RefactorPlan) -> String { + let request = &plan.request; + match request.operation { + RefactorOperation::RenameColumn => format!( + "Rename column {}.{} -> {}", + request.model_name, + request.column_name.as_deref().unwrap_or("None"), + request.new_name + ), + RefactorOperation::MoveModel => format!( + "Move model {} -> {}", + request.model_name, + request.destination.as_deref().unwrap_or("None") + ), + RefactorOperation::RenameModel => { + format!( + "Rename model {} -> {}", + request.model_name, request.new_name + ) + } + } +} + +pub(crate) fn verb(plan: &RefactorPlan) -> &'static str { + if plan.request.operation == RefactorOperation::MoveModel { + "move" + } else { + "rename" + } +} + +/// Edits ordered by start offset, as both outputs list them. +pub(crate) fn sorted_edits(change: &FileChange) -> Vec<&TextEdit> { + let mut edits: Vec<&TextEdit> = change.edits.iter().collect(); + edits.sort_by_key(|edit| edit.start); + edits +} + +pub(crate) fn change_lines(style: CliStyle, changes: &[FileChange]) -> Vec { + let mut lines: Vec = Vec::new(); + for (index, change) in changes.iter().enumerate() { + let last: bool = index + 1 == changes.len(); + let moved: String = if change.moved() { + style.muted(&format!(" (moved from {})", change.original_path)) + } else { + String::new() + }; + lines.push(format!( + "{} {}{moved}", + tree_connector(style, last), + change.path + )); + let prefix: String = style.muted(if last { BLANK_PREFIX } else { CONTINUE_PREFIX }); + let edits: Vec<&TextEdit> = sorted_edits(change); + for (edit_index, edit) in edits.iter().enumerate() { + let connector: String = tree_connector(style, edit_index + 1 == edits.len()); + let position: String = + left_justified(&format!("{}:{}", edit.line, edit.column), POSITION_WIDTH); + let kind: String = left_justified(edit.kind.value(), KIND_WIDTH); + lines.push(if edit.before.is_empty() { + format!( + "{prefix}{connector} {position}{kind}+ {}", + excerpt(&edit.after) + ) + } else { + format!( + "{prefix}{connector} {position}{kind}{} -> {}", + excerpt(&edit.before), + excerpt(&edit.after) + ) + }); + } + } + lines +} + +pub(crate) fn migration_lines(style: CliStyle, migrations: &[MigrationDeclaration]) -> Vec { + migrations + .iter() + .enumerate() + .map(|(index, item)| { + format!( + "{} {} {} {}", + tree_connector(style, index + 1 == migrations.len()), + style.object_name(&item.model_name), + item.declaration, + style.muted(&item.reason) + ) + }) + .collect() +} + +pub(crate) fn position(location: &ManualLocation) -> String { + match location.line { + None => location.path.clone(), + Some(line) => format!( + "{}:{line}:{}", + location.path, + location.column.filter(|column| *column != 0).unwrap_or(1) + ), + } +} + +pub(crate) fn location_lines(style: CliStyle, locations: &[&ManualLocation]) -> Vec { + locations + .iter() + .enumerate() + .map(|(index, item)| { + format!( + "{} {} {}", + tree_connector(style, index + 1 == locations.len()), + position(item), + item.reason + ) + }) + .collect() +} + +pub(crate) fn diagnostic_lines(style: CliStyle, diagnostics: &[RefactorDiagnostic]) -> Vec { + let mut lines: Vec = Vec::new(); + for (index, diagnostic) in diagnostics.iter().enumerate() { + let mut location: String = String::new(); + if let Some(path) = &diagnostic.path { + let line: String = match diagnostic.line { + Some(line) if line != 0 => line.to_string(), + _ => String::new(), + }; + location = format!("{path}:{line} "); + } + lines.push(format!( + "{} {} {location}{}", + tree_connector(style, index + 1 == diagnostics.len()), + diagnostic.code, + first_line(&diagnostic.message) + )); + } + lines +} + +pub(crate) fn layer_move_lines(style: CliStyle, layer_move: &LayerMove) -> Vec { + let folder: String = match (&layer_move.folder, &layer_move.layer_folder) { + (Some(folder), _) => format!("under {folder}/"), + (None, Some(layer_folder)) => format!("under a layer folder named {layer_folder}/"), + (None, None) => "under another layer's folder".to_owned(), + }; + let note: String = if layer_move.folder.is_some() { + "(change the folder if you want it somewhere else)".to_owned() + } else { + format!("(replace {FOLDER_PLACEHOLDER}/ with the folder the model should live in)") + }; + vec![ + format!( + "{} belongs {folder}", + style.object_name(&layer_move.new_name) + ), + " Rename and move in one step:".to_owned(), + format!(" {}", style.command(&layer_move.command)), + format!(" {}", style.muted(¬e)), + ] +} + +pub(crate) fn status_line(style: CliStyle, plan: &RefactorPlan, status: RefactorStatus) -> String { + let files: String = format_count_noun( + i64::try_from(plan.changes.len()).unwrap_or(i64::MAX), + "file", + None, + ); + match status { + RefactorStatus::Applied => style.success_strong(&format!("Compiled: ok, {files} changed")), + RefactorStatus::DryRun => style.success_strong(&format!( + "Dry run: compiles, {files} would change; nothing written" + )), + RefactorStatus::CompileFailed => style.error_strong("Compiled: failed; no files changed"), + RefactorStatus::Refused => style.error_strong("Refused; no files changed"), + } +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs b/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs new file mode 100644 index 0000000000..6237dd184e --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs @@ -0,0 +1,61 @@ +//! Command lines quoted for the shell of the platform they are printed on. + +/// Python's `shlex.quote`. +fn shell_quote(text: &str) -> String { + if text.is_empty() { + return "''".to_owned(); + } + let safe: bool = text + .chars() + .all(|character| character.is_ascii_alphanumeric() || "_@%+=:,./-".contains(character)); + if safe { + return text.to_owned(); + } + format!("'{}'", text.replace('\'', "'\"'\"'")) +} + +/// Python's `subprocess.list2cmdline` for one argument. +fn windows_quote(argument: &str) -> String { + let mut quoted: String = String::new(); + let needs_quotes: bool = argument.is_empty() || argument.contains([' ', '\t']); + if needs_quotes { + quoted.push('"'); + } + let mut backslashes: usize = 0; + for character in argument.chars() { + match character { + '\\' => backslashes += 1, + '"' => { + quoted.push_str(&"\\".repeat(backslashes * 2)); + backslashes = 0; + quoted.push_str("\\\""); + } + _ => { + quoted.push_str(&"\\".repeat(backslashes)); + backslashes = 0; + quoted.push(character); + } + } + } + quoted.push_str(&"\\".repeat(backslashes)); + if needs_quotes { + quoted.push_str(&"\\".repeat(backslashes)); + quoted.push('"'); + } + quoted +} + +/// `shlex.join`, or `subprocess.list2cmdline` on Windows. +pub(crate) fn command_line(arguments: &[String], windows: bool) -> String { + arguments + .iter() + .map(|argument| { + if windows { + windows_quote(argument) + } else { + shell_quote(argument) + } + }) + .collect::>() + .join(" ") +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/_helpers/text.rs b/crates/sqlbuild-refactor/src/refactor_command/_helpers/text.rs new file mode 100644 index 0000000000..9048452088 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/_helpers/text.rs @@ -0,0 +1,62 @@ +//! Small text helpers shared by the refactoring output. + +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::refactor_command::constants::{EXCERPT_LIMIT, TREE_BRANCH_GLYPH, TREE_LAST_GLYPH}; + +/// A dim branch, or the last branch of a group. +pub(crate) fn tree_connector(style: CliStyle, last: bool) -> String { + style.muted(if last { + TREE_LAST_GLYPH + } else { + TREE_BRANCH_GLYPH + }) +} + +/// Python's `str.isspace` for one character. +fn python_space(character: char) -> bool { + character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character) +} + +/// The text on one line with runs of whitespace collapsed, cut to the excerpt limit. +pub(crate) fn excerpt(text: &str) -> String { + let single: String = text + .split(python_space) + .filter(|part| !part.is_empty()) + .collect::>() + .join(" "); + if single.chars().count() <= EXCERPT_LIMIT { + return single; + } + let mut cut: String = single.chars().take(EXCERPT_LIMIT - 1).collect(); + cut.push('\u{2026}'); + cut +} + +/// `text.ljust(width)` counting code points. +pub(crate) fn left_justified(text: &str, width: usize) -> String { + let padding: usize = width.saturating_sub(text.chars().count()); + format!("{text}{}", " ".repeat(padding)) +} + +/// The first line of `text` as Python's `str.splitlines()` splits it. +pub(crate) fn first_line(text: &str) -> &str { + let end: usize = text + .char_indices() + .find(|(_, character)| { + matches!( + character, + '\n' | '\r' + | '\u{0b}' + | '\u{0c}' + | '\u{1c}' + | '\u{1d}' + | '\u{1e}' + | '\u{85}' + | '\u{2028}' + | '\u{2029}' + ) + }) + .map_or(text.len(), |(index, _)| index); + &text[..end] +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/constants.rs b/crates/sqlbuild-refactor/src/refactor_command/constants.rs new file mode 100644 index 0000000000..4d89a600db --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/constants.rs @@ -0,0 +1,46 @@ +//! Labels, layouts and layer folders of `sqb rename` and `sqb mv` output. + +pub(crate) const RENAME_USAGE: &str = + "write the target as or .; the model: and column: prefixes are optional"; +pub(crate) const MOVE_USAGE: &str = "write the target as ; the model: prefix is optional"; +pub(crate) const RENAME_SCOPE: &str = "sqb rename renames models and model columns"; +pub(crate) const MOVE_SCOPE: &str = "sqb mv moves models"; +pub(crate) const TARGET_ERROR_CODE: &str = "C954"; +pub(crate) const COLUMN_TARGET_KIND: &str = "column"; +pub(crate) const COLUMN_TARGET_SEPARATOR: char = '.'; +pub(crate) const MODEL_RESOURCE_TYPE: &str = "model"; +pub(crate) const CONTINUE_PREFIX: &str = "\u{2502} "; +pub(crate) const BLANK_PREFIX: &str = " "; +pub(crate) const POSITION_WIDTH: usize = 8; +pub(crate) const KIND_WIDTH: usize = 10; +pub(crate) const EXCERPT_LIMIT: usize = 60; +pub(crate) const TREE_BRANCH_GLYPH: &str = "\u{251c}\u{2500}\u{2500}"; +pub(crate) const TREE_LAST_GLYPH: &str = "\u{2514}\u{2500}\u{2500}"; +pub(crate) const FOLDER_PLACEHOLDER: &str = ""; +pub(crate) const PROJECT_DIR_OPTION: &str = "--project-dir"; +pub(crate) const SQL_FILE_SUFFIX: &str = ".sql"; +pub(crate) const LAYER_FOLDER_RULE_CODES: [&str; 2] = ["SQBRPROJECT102", "SQBRPROJECT105"]; +pub(crate) const MODEL_NAME_LAYER_SEPARATOR: &str = "__"; +pub(crate) const MODEL_NAME_LAYER_INDEX: usize = 1; +pub(crate) const MODEL_NAME_PART_COUNTS: [usize; 2] = [3, 4]; +/// The folder each name layer belongs under. +pub(crate) const LAYER_FOLDERS: [(&str, &[&str]); 7] = [ + ("stg", &["staging"]), + ("stg_v", &["staging"]), + ("int_clean", &["intermediate", "clean"]), + ("int_enriched", &["intermediate", "enriched"]), + ("int_v", &["intermediate", "enriched"]), + ("mart", &["mart"]), + ("mart_v", &["mart"]), +]; +/// The folder each name layer's models sit in, as one contiguous run of path parts. +pub(crate) const CONTIGUOUS_LAYER_FOLDERS: [(&str, &[&str]); 7] = [ + ("stg", &["staging"]), + ("stg_v", &["staging"]), + ("int_clean", &["intermediate", "clean"]), + ("int_enriched", &["intermediate", "enriched"]), + ("int_v", &["intermediate"]), + ("mart", &["mart"]), + ("mart_v", &["mart"]), +]; +pub(crate) const CURRENT_DIRECTORY: &str = "."; diff --git a/crates/sqlbuild-refactor/src/refactor_command/main/layer_move_suggestion.rs b/crates/sqlbuild-refactor/src/refactor_command/main/layer_move_suggestion.rs new file mode 100644 index 0000000000..ea4e0841c4 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/main/layer_move_suggestion.rs @@ -0,0 +1,67 @@ +//! Suggest the one-step `sqb mv` when a rename fails only on the layer-folder rules. + +use crate::refactor_command::_helpers::layer_folders::{ + folder_of, mirrored_folder, name_layer, path_parts, +}; +use crate::refactor_command::_helpers::quoting::command_line; +use crate::refactor_command::constants::{ + FOLDER_PLACEHOLDER, LAYER_FOLDER_RULE_CODES, LAYER_FOLDERS, PROJECT_DIR_OPTION, SQL_FILE_SUFFIX, +}; +use crate::refactor_command::models::{LayerMove, RefactorDiagnostic}; +use crate::refactoring::models::{FileChange, RefactorOperation, RefactorPlan}; + +/// The `sqb mv` that renames and moves in one step, or none for any other failure. +#[must_use] +pub fn layer_move_suggestion( + plan: &RefactorPlan, + diagnostics: &[RefactorDiagnostic], + (project_dir, windows): (Option<&str>, bool), +) -> Option { + let renamed: &FileChange = plan.changes.iter().find(|change| change.moved())?; + if plan.request.operation != RefactorOperation::RenameModel + || diagnostics.is_empty() + || diagnostics.iter().any(|diagnostic| { + !LAYER_FOLDER_RULE_CODES.contains(&diagnostic.code.as_str()) + || diagnostic.path.as_deref() != Some(renamed.path.as_str()) + }) + { + return None; + } + let new_name: &str = &plan.request.new_name; + let layer_folder: Option<&[&str]> = + name_layer(new_name).and_then(|layer| folder_of(&LAYER_FOLDERS, layer)); + let mirrored: Option<(Vec, Vec)> = layer_folder.and_then(|layer_folder| { + mirrored_folder( + &renamed.original_path, + name_layer(&plan.request.model_name), + layer_folder, + ) + }); + let mut folders: Vec = match &mirrored { + Some((head, tail)) => head.iter().chain(tail).cloned().collect(), + None => vec![ + path_parts(&renamed.original_path) + .first() + .map(|part| (*part).to_owned()) + .unwrap_or_default(), + FOLDER_PLACEHOLDER.to_owned(), + ], + }; + folders.push(format!("{new_name}{SQL_FILE_SUFFIX}")); + let mut arguments: Vec = vec![ + "sqb".to_owned(), + "mv".to_owned(), + plan.request.model_name.clone(), + folders.join("/"), + ]; + if let Some(project_dir) = project_dir { + arguments.push(PROJECT_DIR_OPTION.to_owned()); + arguments.push(project_dir.to_owned()); + } + Some(LayerMove { + new_name: new_name.to_owned(), + folder: mirrored.map(|(head, _)| head.join("/")), + layer_folder: layer_folder.map(|folder| folder.join("/")), + command: command_line(&arguments, windows), + }) +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/main/mod.rs b/crates/sqlbuild-refactor/src/refactor_command/main/mod.rs new file mode 100644 index 0000000000..288df1512f --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/main/mod.rs @@ -0,0 +1,4 @@ +pub mod layer_move_suggestion; +pub mod refactor_request_for_target; +pub mod render_refactor_json; +pub mod render_refactor_text; diff --git a/crates/sqlbuild-refactor/src/refactor_command/main/refactor_request_for_target.rs b/crates/sqlbuild-refactor/src/refactor_command/main/refactor_request_for_target.rs new file mode 100644 index 0000000000..23451b30db --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/main/refactor_request_for_target.rs @@ -0,0 +1,109 @@ +//! Read the target of `sqb rename` and `sqb mv` the way `sqb lineage` reads targets. + +use sqlbuild_analysis::lineage_command::main::resolve_lineage_target::resolve_lineage_target; +use sqlbuild_analysis::lineage_command::models::{LineageKey, LineageTarget}; + +use crate::refactor_command::constants::{ + COLUMN_TARGET_KIND, COLUMN_TARGET_SEPARATOR, MODEL_RESOURCE_TYPE, MOVE_SCOPE, MOVE_USAGE, + RENAME_SCOPE, RENAME_USAGE, TARGET_ERROR_CODE, +}; +use crate::refactor_command::models::RefactorTargetRequest; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{RefactorOperation, RefactorRequest}; + +/// The planner request for a `[kind:]` or `[kind:].` target. +pub fn refactor_request_for_target( + request: &RefactorTargetRequest, + lookup: &dyn Fn(&str) -> Option, +) -> Result { + let raw: &str = request.target.trim(); + let target: LineageTarget = resolve_lineage_target(raw, lookup); + check_target(raw, &target, request.is_move)?; + let model_name: String = target + .name + .split_once(COLUMN_TARGET_SEPARATOR) + .map_or(target.name.as_str(), |(model, _)| model) + .to_owned(); + if request.is_move { + return Ok(RefactorRequest { + operation: RefactorOperation::MoveModel, + model_name, + new_name: String::new(), + column_name: None, + destination: Some(request.destination.clone().unwrap_or_default()), + cascade: false, + }); + } + let new_name: String = request + .new_name + .as_deref() + .unwrap_or_default() + .trim() + .to_owned(); + let Some(column_name) = target.column_name else { + if request.cascade { + return Err(RefactorError::input( + TARGET_ERROR_CODE, + "--cascade applies to column renames only", + None, + )); + } + return Ok(RefactorRequest { + operation: RefactorOperation::RenameModel, + model_name, + new_name, + column_name: None, + destination: None, + cascade: false, + }); + }; + Ok(RefactorRequest { + operation: RefactorOperation::RenameColumn, + model_name, + new_name, + column_name: Some(column_name), + destination: None, + cascade: request.cascade, + }) +} + +fn check_target(raw: &str, target: &LineageTarget, is_move: bool) -> Result<(), RefactorError> { + let verb: &str = if is_move { "move" } else { "rename" }; + let usage: Option<&str> = Some(if is_move { MOVE_USAGE } else { RENAME_USAGE }); + let fail = |message: String| RefactorError::input(TARGET_ERROR_CODE, message, usage); + if target.name.is_empty() || target.column_name.as_deref() == Some("") { + return Err(fail(format!("cannot {verb} '{raw}'"))); + } + if let Some(key) = &target.key { + if key.resource_type != MODEL_RESOURCE_TYPE { + let scope: &str = if is_move { MOVE_SCOPE } else { RENAME_SCOPE }; + return Err(fail(format!( + "cannot {verb} '{raw}': {} is a {}; {scope}", + key.name, key.resource_type + ))); + } + if !target.kind_matches { + let kind: &str = target.kind.as_deref().unwrap_or_default(); + let found: String = if kind == COLUMN_TARGET_KIND { + format!( + "column: needs ., and {} names a model", + key.name + ) + } else { + format!("{} is a {}, not a {kind}", key.name, key.resource_type) + }; + return Err(fail(format!("cannot {verb} '{raw}': {found}"))); + } + } + if is_move && target.column_name.is_some() { + return Err(RefactorError::input( + TARGET_ERROR_CODE, + format!("sqb mv moves models; '{raw}' names a column"), + Some(&format!( + "rename a column with sqb rename {} ", + target.name + )), + )); + } + Ok(()) +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/main/render_refactor_json.rs b/crates/sqlbuild-refactor/src/refactor_command/main/render_refactor_json.rs new file mode 100644 index 0000000000..58f53d4544 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/main/render_refactor_json.rs @@ -0,0 +1,101 @@ +//! The JSON output of `sqb rename` and `sqb mv`. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::refactor_command::_helpers::json::{ + count, file, integer, location, object, operation_value, optional_text, text, +}; +use crate::refactor_command::_helpers::output_text::refactor_target; +use crate::refactor_command::models::{RefactorDiagnostic, RefactorStatus}; +use crate::refactoring::models::RefactorPlan; + +/// The same facts as the text output, as stable JSON with a trailing newline. +pub fn render_refactor_json( + plan: &RefactorPlan, + status: RefactorStatus, + diagnostics: &[RefactorDiagnostic], +) -> Result { + let request = &plan.request; + let changed: usize = if status == RefactorStatus::Applied { + plan.changes.len() + } else { + 0 + }; + let payload: JsonValue = object(vec![ + ("operation", text(operation_value(plan))), + ("target", text(&refactor_target(plan))), + ("new_name", text(&request.new_name)), + ("destination", optional_text(request.destination.as_deref())), + ("cascade", JsonValue::Bool(request.cascade)), + ("status", text(status.value())), + ("files_changed", count(changed)), + ( + "files", + JsonValue::Array(plan.changes.iter().map(file).collect()), + ), + ( + "migrations", + JsonValue::Array( + plan.migrations + .iter() + .map(|migration| { + object(vec![ + ("model", text(&migration.model_name)), + ("declaration", text(&migration.declaration)), + ("reason", text(&migration.reason)), + ]) + }) + .collect(), + ), + ), + ( + "renamed_columns", + JsonValue::Array( + plan.renamed_columns + .iter() + .map(|(model, old, new)| { + object(vec![ + ("model", text(model)), + ("from", text(old)), + ("to", text(new)), + ]) + }) + .collect(), + ), + ), + ( + "manual", + JsonValue::Array(plan.manual.iter().map(location).collect()), + ), + ( + "blocking", + JsonValue::Array(plan.blocking.iter().map(location).collect()), + ), + ( + "compile", + object(vec![ + ("ok", JsonValue::Bool(diagnostics.is_empty())), + ( + "errors", + JsonValue::Array( + diagnostics + .iter() + .map(|diagnostic| { + object(vec![ + ("code", text(&diagnostic.code)), + ("message", text(&diagnostic.message)), + ("path", optional_text(diagnostic.path.as_deref())), + ("line", integer(diagnostic.line)), + ]) + }) + .collect(), + ), + ), + ]), + ), + ]); + let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))); + Ok(format!("{}\n", dumps(&payload, &dialect)?)) +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/main/render_refactor_text.rs b/crates/sqlbuild-refactor/src/refactor_command/main/render_refactor_text.rs new file mode 100644 index 0000000000..471e5903f4 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/main/render_refactor_text.rs @@ -0,0 +1,52 @@ +//! The human output of `sqb rename` and `sqb mv`. + +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::refactor_command::_helpers::output_text::{ + change_lines, diagnostic_lines, heading, layer_move_lines, location_lines, migration_lines, + refactor_target, status_line, verb, +}; +use crate::refactor_command::models::{LayerMove, RefactorDiagnostic, RefactorStatus}; +use crate::refactoring::models::{ManualLocation, RefactorPlan}; + +/// The edit tree, manual locations, compile errors, and one terminal status line. +#[must_use] +pub fn render_refactor_text( + plan: &RefactorPlan, + (status, diagnostics): (RefactorStatus, &[RefactorDiagnostic]), + layer_move: Option<&LayerMove>, + style: CliStyle, +) -> String { + let mut lines: Vec = vec![style.title(&heading(plan))]; + let blocked: bool = status == RefactorStatus::Refused; + if !blocked || (plan.blocking.is_empty() && plan.manual.is_empty()) { + lines.extend(change_lines(style, &plan.changes)); + } + if !plan.migrations.is_empty() && !blocked { + lines.push(style.section("Migrations")); + lines.extend(migration_lines(style, &plan.migrations)); + } + if blocked { + lines.push(style.error_strong(&format!( + "Cannot {} {} automatically", + verb(plan), + refactor_target(plan) + ))); + let locations: Vec<&ManualLocation> = plan.blocking.iter().chain(&plan.manual).collect(); + lines.extend(location_lines(style, &locations)); + if let Some(help) = &plan.help + && !plan.manual.is_empty() + { + lines.push(help.clone()); + } + } + if !diagnostics.is_empty() { + lines.push(style.error_strong("Compile errors in the edited project")); + lines.extend(diagnostic_lines(style, diagnostics)); + } + if let Some(layer_move) = layer_move { + lines.extend(layer_move_lines(style, layer_move)); + } + lines.push(status_line(style, plan, status)); + format!("{}\n", lines.join("\n")) +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/mod.rs b/crates/sqlbuild-refactor/src/refactor_command/mod.rs new file mode 100644 index 0000000000..ee98d9836b --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/mod.rs @@ -0,0 +1,9 @@ +//! `sqb rename` and `sqb mv`: target reading, the layer-move suggestion, and text and JSON output. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-refactor/src/refactor_command/models.rs b/crates/sqlbuild-refactor/src/refactor_command/models.rs new file mode 100644 index 0000000000..7defc74f34 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/models.rs @@ -0,0 +1,52 @@ +//! Requests, outcomes and output facts of `sqb rename` and `sqb mv`. + +/// Terminal state of one refactoring command. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RefactorStatus { + Applied, + DryRun, + Refused, + CompileFailed, +} + +impl RefactorStatus { + /// The status as JSON output spells it. + #[must_use] + pub fn value(self) -> &'static str { + match self { + Self::Applied => "applied", + Self::DryRun => "dry_run", + Self::Refused => "refused", + Self::CompileFailed => "compile_failed", + } + } +} + +/// The CLI target of `sqb rename` or `sqb mv`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RefactorTargetRequest { + pub is_move: bool, + pub target: String, + pub new_name: Option, + pub destination: Option, + pub cascade: bool, +} + +/// One compile error of the edited project. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RefactorDiagnostic { + pub code: String, + pub message: String, + /// The path in POSIX form. + pub path: Option, + pub line: Option, +} + +/// The one-step `sqb mv` for a rename refused only by the layer-folder rules. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LayerMove { + pub new_name: String, + pub folder: Option, + pub layer_folder: Option, + pub command: String, +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/tests.rs b/crates/sqlbuild-refactor/src/refactor_command/tests.rs new file mode 100644 index 0000000000..e8b2db7cab --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_layer_move.rs"] +mod layer_move; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-refactor/src/refactor_command/tests/helpers.rs b/crates/sqlbuild-refactor/src/refactor_command/tests/helpers.rs new file mode 100644 index 0000000000..b991b82e9a --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/tests/helpers.rs @@ -0,0 +1,40 @@ +use crate::refactor_command::models::RefactorDiagnostic; +use crate::refactoring::models::{FileChange, RefactorOperation, RefactorPlan, RefactorRequest}; + +/// A model rename plan whose file keeps its folder. +pub(super) fn rename_plan(old_path: &str, new_name: &str) -> RefactorPlan { + let (folder, file) = old_path.rsplit_once('/').unwrap_or(("", old_path)); + RefactorPlan { + request: RefactorRequest { + operation: RefactorOperation::RenameModel, + model_name: file.trim_end_matches(".sql").to_owned(), + new_name: new_name.to_owned(), + column_name: None, + destination: None, + cascade: false, + }, + changes: vec![FileChange { + path: format!("{folder}/{new_name}.sql"), + original_path: old_path.to_owned(), + edits: Vec::new(), + }], + manual: Vec::new(), + blocking: Vec::new(), + migrations: Vec::new(), + renamed_columns: Vec::new(), + help: None, + } +} + +/// One compile error per rule code, reported on `path`. +pub(super) fn rule_diagnostics(codes: &[&str], path: &str) -> Vec { + codes + .iter() + .map(|code| RefactorDiagnostic { + code: (*code).to_owned(), + message: format!("{code} failed"), + path: Some(path.to_owned()), + line: Some(1), + }) + .collect() +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/tests/test_layer_move.rs b/crates/sqlbuild-refactor/src/refactor_command/tests/test_layer_move.rs new file mode 100644 index 0000000000..4b3a867a8b --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/tests/test_layer_move.rs @@ -0,0 +1,177 @@ +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::refactor_command::main::layer_move_suggestion::layer_move_suggestion; +use crate::refactor_command::main::render_refactor_text::render_refactor_text; +use crate::refactor_command::models::{LayerMove, RefactorDiagnostic, RefactorStatus}; +use crate::refactor_command::tests::helpers::{rename_plan, rule_diagnostics}; +use crate::refactor_command::tests::test_types::{LayerMoveTestCase, RenderTestCase}; +use crate::refactoring::models::RefactorPlan; + +const LAYER_CODES: &[&str] = &["SQBRPROJECT102", "SQBRPROJECT105"]; +const ENRICHED_COMMAND: &str = "sqb mv orders__int_clean__lines__web \ + models/orders/intermediate/enriched/web/orders__int_enriched__lines__web.sql"; + +#[test] +fn given_rename_compile_errors_when_suggesting_layer_move_then_one_step_mv_is_exact() { + let test_cases = [ + LayerMoveTestCase { + description: "int_clean to int_enriched keeps domain and sub-folders", + old_path: "models/orders/intermediate/clean/web/orders__int_clean__lines__web.sql", + new_name: "orders__int_enriched__lines__web", + codes: LAYER_CODES, + project_dir: None, + windows: false, + expected_command: Some(ENRICHED_COMMAND), + expected_folder: Some("models/orders/intermediate/enriched"), + }, + LayerMoveTestCase { + description: "a bare intermediate folder becomes the int_clean folder", + old_path: "models/orders/intermediate/orders__int_v__totals.sql", + new_name: "orders__int_clean__totals", + codes: &["SQBRPROJECT102"], + project_dir: None, + windows: false, + expected_command: Some( + "sqb mv orders__int_v__totals \ + models/orders/intermediate/clean/orders__int_clean__totals.sql", + ), + expected_folder: Some("models/orders/intermediate/clean"), + }, + LayerMoveTestCase { + description: "only the current layer's own folder is replaced", + old_path: "models/mart/orders/staging/orders__stg__order_lines_x.sql", + new_name: "orders__int_clean__order_lines_x", + codes: &["SQBRPROJECT102"], + project_dir: None, + windows: false, + expected_command: Some( + "sqb mv orders__stg__order_lines_x \ + models/mart/orders/intermediate/clean/orders__int_clean__order_lines_x.sql", + ), + expected_folder: Some("models/mart/orders/intermediate/clean"), + }, + LayerMoveTestCase { + description: "no layer folder in the current path uses a placeholder", + old_path: "models/orders/web/order_lines.sql", + new_name: "orders__stg__order_lines", + codes: LAYER_CODES, + project_dir: None, + windows: false, + expected_command: Some( + "sqb mv order_lines 'models//orders__stg__order_lines.sql'", + ), + expected_folder: None, + }, + LayerMoveTestCase { + description: "a layer without a folder mapping uses a placeholder", + old_path: "models/orders/intermediate/clean/orders__int_clean__totals.sql", + new_name: "orders__int_archive__totals", + codes: LAYER_CODES, + project_dir: None, + windows: false, + expected_command: Some( + "sqb mv orders__int_clean__totals \ + 'models//orders__int_archive__totals.sql'", + ), + expected_folder: None, + }, + LayerMoveTestCase { + description: "another rule failing as well gets no suggestion", + old_path: "models/orders/intermediate/clean/orders__int_clean__totals.sql", + new_name: "orders__int_enriched__totals", + codes: &["SQBRPROJECT105", "SQBRMODEL103"], + project_dir: None, + windows: false, + expected_command: None, + expected_folder: None, + }, + LayerMoveTestCase { + description: "posix shells get POSIX quoting", + old_path: "models/orders/intermediate/clean/web/orders__int_clean__lines__web.sql", + new_name: "orders__int_enriched__lines__web", + codes: LAYER_CODES, + project_dir: Some("/srv/order projects/orders"), + windows: false, + expected_command: Some( + "sqb mv orders__int_clean__lines__web \ + models/orders/intermediate/enriched/web/orders__int_enriched__lines__web.sql \ + --project-dir '/srv/order projects/orders'", + ), + expected_folder: Some("models/orders/intermediate/enriched"), + }, + LayerMoveTestCase { + description: "windows gets cmd.exe quoting", + old_path: "models/orders/intermediate/clean/web/orders__int_clean__lines__web.sql", + new_name: "orders__int_enriched__lines__web", + codes: LAYER_CODES, + project_dir: Some("C:\\order projects\\orders"), + windows: true, + expected_command: Some( + "sqb mv orders__int_clean__lines__web \ + models/orders/intermediate/enriched/web/orders__int_enriched__lines__web.sql \ + --project-dir \"C:\\order projects\\orders\"", + ), + expected_folder: Some("models/orders/intermediate/enriched"), + }, + ]; + + for test_case in test_cases { + let plan: RefactorPlan = rename_plan(test_case.old_path, test_case.new_name); + let diagnostics: Vec = + rule_diagnostics(test_case.codes, &plan.changes[0].path); + let suggestion: Option = layer_move_suggestion( + &plan, + &diagnostics, + (test_case.project_dir, test_case.windows), + ); + assert_eq!( + ( + suggestion.as_ref().map(|item| item.command.as_str()), + suggestion.as_ref().and_then(|item| item.folder.as_deref()), + ), + (test_case.expected_command, test_case.expected_folder), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_layer_failure_when_rendering_text_then_suggestion_follows_errors() { + let test_cases = [RenderTestCase { + description: "a refused rename lists errors, the one-step move and the status", + old_path: "models/orders/staging/orders__stg__lines.sql", + new_name: "orders__mart__lines", + codes: &["SQBRPROJECT105"], + expected_text: "Rename model orders__stg__lines -> orders__mart__lines\n\ + \u{2514}\u{2500}\u{2500} models/orders/staging/orders__mart__lines.sql \ + (moved from models/orders/staging/orders__stg__lines.sql)\n\ + Compile errors in the edited project\n\ + \u{2514}\u{2500}\u{2500} SQBRPROJECT105 \ + models/orders/staging/orders__mart__lines.sql:1 SQBRPROJECT105 failed\n\ + orders__mart__lines belongs under models/orders/mart/\n \ + Rename and move in one step:\n \ + sqb mv orders__stg__lines models/orders/mart/orders__mart__lines.sql\n \ + (change the folder if you want it somewhere else)\n\ + Compiled: failed; no files changed\n", + }]; + + for test_case in test_cases { + let plan: RefactorPlan = rename_plan(test_case.old_path, test_case.new_name); + let diagnostics: Vec = + rule_diagnostics(test_case.codes, &plan.changes[0].path); + let suggestion: Option = + layer_move_suggestion(&plan, &diagnostics, (None, false)); + assert_eq!( + render_refactor_text( + &plan, + (RefactorStatus::CompileFailed, &diagnostics), + suggestion.as_ref(), + CliStyle { use_color: false }, + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-refactor/src/refactor_command/tests/test_types.rs b/crates/sqlbuild-refactor/src/refactor_command/tests/test_types.rs new file mode 100644 index 0000000000..78fa819124 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactor_command/tests/test_types.rs @@ -0,0 +1,18 @@ +pub(super) struct LayerMoveTestCase { + pub(super) description: &'static str, + pub(super) old_path: &'static str, + pub(super) new_name: &'static str, + pub(super) codes: &'static [&'static str], + pub(super) project_dir: Option<&'static str>, + pub(super) windows: bool, + pub(super) expected_command: Option<&'static str>, + pub(super) expected_folder: Option<&'static str>, +} + +pub(super) struct RenderTestCase { + pub(super) description: &'static str, + pub(super) old_path: &'static str, + pub(super) new_name: &'static str, + pub(super) codes: &'static [&'static str], + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/_helpers/audits.rs b/crates/sqlbuild-rules/src/dsl_headers/_helpers/audits.rs new file mode 100644 index 0000000000..286ee64e78 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/_helpers/audits.rs @@ -0,0 +1,147 @@ +//! Header audit lists split one item per line when a header line is too long. + +use sqlbuild_core::text::main::python_isspace::python_isspace; + +use crate::dsl_headers::_helpers::quoted::quoted_value_end; +use crate::dsl_headers::_helpers::text::{chars, contains, is_word, lstrip, slice, strip}; +use crate::dsl_headers::constants::{AUDIT_WRAP_MAX_PASSES, AUDITS_KEYWORD, HEADER_INDENT, QUOTES}; + +/// One bracket group inside a header audit list. +struct Group { + open: usize, + close: usize, + depth: usize, + commas: Vec, +} + +/// Whether `audits\s*[` matches at `start` after a word boundary. +fn audits_key_at(text: &[char], start: usize) -> bool { + let keyword: Vec = chars(AUDITS_KEYWORD); + if (start > 0 && is_word(text[start - 1])) || !text[start..].starts_with(&keyword) { + return false; + } + let mut index: usize = start + keyword.len(); + while index < text.len() && python_isspace(text[index]) { + index += 1; + } + text.get(index) == Some(&'[') +} + +/// Where `audits` would start for the bracket at `bracket`. +fn audits_key_start(text: &[char], bracket: usize) -> usize { + let mut start: usize = bracket; + while start > 0 && python_isspace(text[start - 1]) { + start -= 1; + } + start.saturating_sub(AUDITS_KEYWORD.len()) +} + +/// Bracket groups at or inside each `audits [...]` value, outside quoted text. +fn audit_groups(text: &[char]) -> Vec { + let mut groups: Vec = Vec::new(); + let mut open: Vec<(usize, char, Vec)> = Vec::new(); + let mut audit_depths: Vec = Vec::new(); + let mut index: usize = 0; + while index < text.len() { + let character: char = text[index]; + if QUOTES.contains(&character) { + index = quoted_value_end(text, index); + continue; + } + if character == '(' || character == '[' { + if character == '[' && audits_key_at(text, audits_key_start(text, index)) { + audit_depths.push(open.len()); + } + open.push((index, if character == '(' { ')' } else { ']' }, Vec::new())); + } else if open + .last() + .is_some_and(|(_, closer, _)| *closer == character) + { + if let Some((start, _, commas)) = open.pop() { + if audit_depths + .last() + .is_some_and(|depth| open.len() >= *depth) + { + groups.push(Group { + open: start, + close: index, + depth: open.len(), + commas, + }); + } + if audit_depths.last() == Some(&open.len()) { + audit_depths.pop(); + } + } + } else if character == ',' + && let Some((_, _, commas)) = open.last_mut() + { + commas.push(index); + } + index += 1; + } + groups +} + +fn explode(text: &[char], group: &Group, indent: &[char]) -> Vec { + let mut bounds: Vec = vec![group.open]; + bounds.extend(&group.commas); + bounds.push(group.close); + let inner: Vec = [indent, &chars(HEADER_INDENT)].concat(); + let mut exploded: Vec = text[..=group.open].to_vec(); + for pair in bounds.windows(2) { + let item: &[char] = strip(slice(text, pair[0] + 1, pair[1])); + if item.is_empty() { + continue; + } + exploded.push('\n'); + exploded.extend(&inner); + exploded.extend(item); + exploded.push(','); + } + exploded.push('\n'); + exploded.extend(indent); + exploded.extend(&text[group.close..]); + exploded +} + +fn explode_first_long_line(text: &[char], line_width: i64) -> Option> { + let groups: Vec = audit_groups(text); + let width: usize = usize::try_from(line_width).unwrap_or(0); + let mut line_start: usize = 0; + for line in text.split(|character| *character == '\n') { + let line_end: usize = line_start + line.len(); + if i64::try_from(line.len()).unwrap_or(i64::MAX) > line_width && !contains(line, "--") { + let chosen: Option<&Group> = groups + .iter() + .filter(|group| { + line_start <= group.open + && group.close < line_end + && group.close - line_start >= width + && !strip(slice(text, group.open + 1, group.close)).is_empty() + }) + .fold(None, |best: Option<&Group>, group| match best { + Some(best) if best.depth <= group.depth => Some(best), + _ => Some(group), + }); + if let Some(group) = chosen { + let indent: &[char] = &line[..line.len() - lstrip(line).len()]; + return Some(explode(text, group, indent)); + } + } + line_start = line_end + 1; + } + None +} + +/// The header with its long audit-list lines exploded, repeated until none remain. +pub(crate) fn wrap_header(header_text: &[char], line_width: i64) -> Vec { + let mut text: Vec = header_text.to_vec(); + for _ in 0..AUDIT_WRAP_MAX_PASSES { + match explode_first_long_line(&text, line_width) { + Some(exploded) => text = exploded, + None => return text, + } + } + text +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/_helpers/formatting.rs b/crates/sqlbuild-rules/src/dsl_headers/_helpers/formatting.rs new file mode 100644 index 0000000000..2eaed5d2ef --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/_helpers/formatting.rs @@ -0,0 +1,229 @@ +//! Header formatting fixes: trailing spaces, leading comments, description and audit wrapping. + +use sqlbuild_core::text::main::python_splitlines::python_splitlines; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::dsl_headers::_helpers::audits::wrap_header; +use crate::dsl_headers::_helpers::quoted::{ + decode_description, description_value_span, escape_quoted, split_outside_quotes, +}; +use crate::dsl_headers::_helpers::rules::violation_at; +use crate::dsl_headers::_helpers::text::{ + chars, line_count, line_starts, lstrip, rstrip, slice, string, tail, +}; +use crate::dsl_headers::_helpers::wrap::{wrap_description, wrap_header_description}; +use crate::dsl_headers::constants::{ + DESCRIPTION_HEADER_KINDS, DESCRIPTION_KEY, HEADER_INDENT, MODEL_KIND, + RULE_LEADING_COMMENT_DESCRIPTION, +}; +use crate::dsl_headers::main::inner_header_text::inner_header_text; +use crate::dsl_headers::main::scan_headers::scan_headers; +use crate::dsl_headers::models::{ + HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, ParseOutcome, +}; +use crate::dsl_headers::types::HeaderParser; + +pub(crate) fn without_trailing_whitespace(text: &[char], headers: &[HeaderSpan]) -> Vec { + let mut updated: Vec = Vec::with_capacity(text.len()); + let mut cursor: usize = 0; + for header in headers { + updated.extend(slice(text, cursor, header.start)); + let lines: Vec<&[char]> = split_outside_quotes(slice(text, header.start, header.end)) + .into_iter() + .map(rstrip) + .collect(); + updated.extend(lines.join(&'\n')); + cursor = header.end; + } + updated.extend(tail(text, cursor)); + updated +} + +/// The prose of the comment block before the first header, if it is only comments. +fn leading_comment(preamble: &[char]) -> Option { + let preamble: String = string(preamble); + let stripped: &str = python_strip(&preamble); + if stripped.is_empty() { + return None; + } + if let Some(body) = stripped.strip_prefix("/*") { + return body.find("*/").map(|end| body[..end].to_owned()); + } + let mut lines: Vec = Vec::new(); + for raw_line in python_splitlines(&preamble) { + let candidate: &str = python_strip(raw_line); + if let Some(comment) = candidate.strip_prefix("--") { + lines.push(string(lstrip(&chars(comment)))); + continue; + } + if !lines.is_empty() { + break; + } + if !candidate.is_empty() { + return None; + } + } + (!lines.is_empty()).then(|| lines.join("\n")) +} + +/// The comment wrapped as a description, and the fault when it is too long. +fn relocated_description( + comment: &str, + config: HeaderRulesConfig, + (text, header_start): (&[char], usize), +) -> (Vec, Vec) { + let prefix_width: i64 = + i64::try_from(HEADER_INDENT.len() + DESCRIPTION_KEY.len() + 2).unwrap_or(i64::MAX); + let closing_width: i64 = 2; + let description: Vec = wrap_description( + &chars(comment), + (config.line_width - prefix_width - closing_width).max(1), + (config.line_width - closing_width).max(1), + ); + let limit: usize = config.max_description_lines; + if line_count(&description) <= limit { + return (description, Vec::new()); + } + let fault: HeaderViolation = violation_at( + &line_starts(text), + header_start, + (RULE_LEADING_COMMENT_DESCRIPTION, HeaderSeverity::Fault), + ( + format!("Relocated leading comment exceeds {limit} lines"), + format!("Shorten the relocated description to {limit} lines."), + ), + ); + (description, vec![fault]) +} + +/// The `description "..."` line inserted after a model header's opening parenthesis. +fn description_insertion(description: &[char]) -> Vec { + let escaped: String = string(description) + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace("\r\n", "\n"); + chars(&format!( + "\n{HEADER_INDENT}{DESCRIPTION_KEY} \"{escaped}\"," + )) +} + +/// A model header's leading comment moved into its description, with any length fault. +pub(crate) fn relocate_leading_comment( + text: &[char], + config: HeaderRulesConfig, + headers: &[HeaderSpan], + parser: &P, +) -> Result<(Vec, Vec), P::Error> { + let unchanged = || (text.to_vec(), Vec::new()); + let Some(header) = headers.first().filter(|header| header.kind == MODEL_KIND) else { + return Ok(unchanged()); + }; + let Some(comment) = leading_comment(slice(text, 0, header.start)) else { + return Ok(unchanged()); + }; + let header_text: &[char] = slice(text, header.start, header.end); + let values: P::Values = + match parser.parse(&header.kind, &string(inner_header_text(header_text))) { + ParseOutcome::Parsed(values) => values, + ParseOutcome::Rejected(_) | ParseOutcome::Raised(..) => return Ok(unchanged()), + }; + if parser.description(&values)?.is_some() { + return Ok(unchanged()); + } + let (description, faults) = relocated_description(&comment, config, (text, header.start)); + let insertion: Vec = description_insertion(&description); + let mut updated: Vec = match header_text.iter().position(|character| *character == '(') { + Some(open) => [&header_text[..=open], &insertion, &header_text[open + 1..]].concat(), + None => header_text.to_vec(), + }; + updated.extend(tail(text, header.end)); + Ok((updated, faults)) +} + +pub(crate) fn wrap_descriptions( + text: &[char], + headers: &[HeaderSpan], + line_width: i64, + parser: &P, +) -> Result, P::Error> { + let mut updated: Vec = text.to_vec(); + for header in headers.iter().rev() { + if !DESCRIPTION_HEADER_KINDS.contains(&header.kind.as_str()) { + continue; + } + let header_text: Vec = slice(&updated, header.start, header.end).to_vec(); + let Some((start, end)) = description_value_span(&header_text) else { + continue; + }; + let values: P::Values = + match parser.parse(&header.kind, &string(inner_header_text(&header_text))) { + ParseOutcome::Parsed(values) => values, + ParseOutcome::Rejected(_) => continue, + ParseOutcome::Raised(error, _) => return Err(error), + }; + if parser.description(&values)?.is_none() { + continue; + } + let quote: char = header_text[start - 1]; + let wrapped: Vec = wrap_header_description( + &decode_description(slice(&header_text, start, end), quote), + &header_text, + (start, end), + line_width, + ); + let rewritten: Vec = [ + slice(&header_text, 0, start), + &escape_quoted(&wrapped, quote), + tail(&header_text, end), + ] + .concat(); + updated = [ + slice(&updated, 0, header.start), + &rewritten, + tail(&updated, header.end), + ] + .concat(); + } + Ok(updated) +} + +/// Whether two header texts parse to equal values; a header syntax error equals nothing. +fn same_values( + kind: &str, + (left, right): (&[char], &[char]), + parser: &P, +) -> Result { + let parse = |text: &[char]| match parser.parse(kind, &string(inner_header_text(text))) { + ParseOutcome::Parsed(values) => Ok(Some(values)), + ParseOutcome::Rejected(_) => Ok(None), + ParseOutcome::Raised(error, _) => Err(error), + }; + let left: Option = parse(left)?; + let right: Option = parse(right)?; + match (left, right) { + (Some(left), Some(right)) => parser.same(&left, &right), + _ => Ok(false), + } +} + +pub(crate) fn wrap_audits( + text: &[char], + line_width: i64, + parser: &P, +) -> Result, P::Error> { + let mut updated: Vec = text.to_vec(); + for header in scan_headers(text, false).iter().rev() { + let header_text: Vec = slice(&updated, header.start, header.end).to_vec(); + let wrapped: Vec = wrap_header(&header_text, line_width); + if wrapped == header_text || !same_values(&header.kind, (&wrapped, &header_text), parser)? { + continue; + } + updated = [ + slice(&updated, 0, header.start), + &wrapped, + tail(&updated, header.end), + ] + .concat(); + } + Ok(updated) +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/_helpers/mod.rs b/crates/sqlbuild-rules/src/dsl_headers/_helpers/mod.rs new file mode 100644 index 0000000000..a7b7476dc5 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod audits; +pub(crate) mod formatting; +pub(crate) mod quoted; +pub(crate) mod rules; +pub(crate) mod scan; +pub(crate) mod text; +pub(crate) mod wrap; diff --git a/crates/sqlbuild-rules/src/dsl_headers/_helpers/quoted.rs b/crates/sqlbuild-rules/src/dsl_headers/_helpers/quoted.rs new file mode 100644 index 0000000000..8b76cc1b14 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/_helpers/quoted.rs @@ -0,0 +1,175 @@ +//! Quoted header values: their extent, escapes, and lines split outside them. + +use sqlbuild_core::text::main::python_isalnum::python_isalnum; +use sqlbuild_core::text::main::python_isspace::python_isspace; + +use crate::dsl_headers::_helpers::text::{is_alpha, string}; +use crate::dsl_headers::constants::{DESCRIPTION_KEY, ESCAPE, QUOTES}; + +const MAX_CODEPOINT: u32 = 0x0010_FFFF; +const SURROGATES: std::ops::RangeInclusive = 0xD800..=0xDFFF; + +/// The offset just past the quoted value that opens at `start`. +pub(crate) fn quoted_value_end(text: &[char], start: usize) -> usize { + let quote: char = text[start]; + let mut index: usize = start + 1; + while index < text.len() { + if text[index] == ESCAPE && index + 1 < text.len() { + index += 2; + continue; + } + if text[index] == quote { + return index + 1; + } + index += 1; + } + text.len() +} + +/// The span of the top-level `description` string value inside the header parentheses. +pub(crate) fn description_value_span(text: &[char]) -> Option<(usize, usize)> { + let mut delimiters: Vec = Vec::new(); + let mut index: usize = 0; + while index < text.len() { + let character: char = text[index]; + if QUOTES.contains(&character) { + index = quoted_value_end(text, index); + continue; + } + let closing: Option = match character { + '(' => Some(')'), + '[' => Some(']'), + '{' => Some('}'), + _ => None, + }; + if let Some(closing) = closing { + delimiters.push(closing); + index += 1; + continue; + } + if delimiters.last() == Some(&character) { + delimiters.pop(); + index += 1; + continue; + } + if delimiters == [')'] && (is_alpha(character) || character == '_') { + let mut word_end: usize = index + 1; + while word_end < text.len() && (python_isalnum(text[word_end]) || text[word_end] == '_') + { + word_end += 1; + } + if string(&text[index..word_end]).to_lowercase() == DESCRIPTION_KEY { + let mut quote_start: usize = word_end; + while quote_start < text.len() && python_isspace(text[quote_start]) { + quote_start += 1; + } + if quote_start < text.len() && QUOTES.contains(&text[quote_start]) { + let quote_end: usize = quoted_value_end(text, quote_start); + return Some((quote_start + 1, quote_end - 1)); + } + } + index = word_end; + continue; + } + index += 1; + } + None +} + +/// The text of a quoted description value with its escapes decoded. +pub(crate) fn decode_description(value: &[char], quote: char) -> Vec { + let mut decoded: Vec = Vec::with_capacity(value.len()); + let mut index: usize = 0; + while index < value.len() { + let character: char = value[index]; + if character != ESCAPE || index + 1 >= value.len() { + decoded.push(character); + index += 1; + continue; + } + let escaped: char = value[index + 1]; + if escaped == ESCAPE || escaped == quote { + decoded.push(escaped); + index += 2; + continue; + } + let simple: Option = match escaped { + 'b' => Some('\u{8}'), + 'f' => Some('\u{c}'), + 'n' => Some('\n'), + 'r' => Some('\r'), + 't' => Some('\t'), + _ => None, + }; + if let Some(simple) = simple { + decoded.push(simple); + index += 2; + continue; + } + let width: usize = match escaped { + 'u' => 4, + 'U' => 8, + _ => 0, + }; + let digits: &[char] = super::text::slice(value, index + 2, index + 2 + width); + if width > 0 && digits.len() == width && digits.iter().all(char::is_ascii_hexdigit) { + let codepoint: Option = match u32::from_str_radix(&string(digits), 16) { + Ok(codepoint) if codepoint <= MAX_CODEPOINT && !SURROGATES.contains(&codepoint) => { + char::from_u32(codepoint) + } + Ok(_) | Err(_) => None, + }; + if let Some(decoded_character) = codepoint { + decoded.push(decoded_character); + index += width + 2; + continue; + } + } + decoded.extend([ESCAPE, escaped]); + index += 2; + } + decoded +} + +/// `value` with backslashes and `quote` escaped for a quoted header value. +pub(crate) fn escape_quoted(value: &[char], quote: char) -> Vec { + let mut escaped: Vec = Vec::with_capacity(value.len()); + for character in value { + if *character == ESCAPE || *character == quote { + escaped.push(ESCAPE); + } + escaped.push(*character); + } + escaped +} + +/// `text` split on newlines that are outside quoted values. +pub(crate) fn split_outside_quotes(text: &[char]) -> Vec<&[char]> { + let mut lines: Vec<&[char]> = Vec::new(); + let mut line_start: usize = 0; + let mut in_quote: Option = None; + let mut index: usize = 0; + while index < text.len() { + let character: char = text[index]; + if let Some(quote) = in_quote { + if character == ESCAPE && index + 1 < text.len() { + index += 2; + continue; + } + if character == quote { + in_quote = None; + } + index += 1; + continue; + } + if QUOTES.contains(&character) { + in_quote = Some(character); + } else if character == '\n' { + lines.push(&text[line_start..index]); + line_start = index + 1; + } + index += 1; + } + lines.push(&text[line_start..]); + lines +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/_helpers/rules.rs b/crates/sqlbuild-rules/src/dsl_headers/_helpers/rules.rs new file mode 100644 index 0000000000..a2005dd3d6 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/_helpers/rules.rs @@ -0,0 +1,137 @@ +//! The header rules of one header: parseable values, a short present description, no trailing spaces. + +use crate::dsl_headers::_helpers::quoted::{ + decode_description, description_value_span, split_outside_quotes, +}; +use crate::dsl_headers::_helpers::text::{line_count, position, rstrip, slice, string, strip}; +use crate::dsl_headers::_helpers::wrap::wrap_header_description; +use crate::dsl_headers::constants::{ + DESCRIPTION_HEADER_KINDS, DESCRIPTION_REQUIRED_HEADER_KINDS, RULE_DESCRIPTION_LENGTH, + RULE_DESCRIPTION_PRESENT, RULE_HEADER_PARSE, RULE_HEADER_WHITESPACE, +}; +use crate::dsl_headers::main::inner_header_text::inner_header_text; +use crate::dsl_headers::models::{ + HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, ParseFailure, ParseOutcome, +}; +use crate::dsl_headers::types::HeaderParser; + +/// One violation reported at the start of `header`. +pub(crate) fn violation_at( + starts: &[usize], + offset: usize, + (code, severity): (&'static str, HeaderSeverity), + (message, remediation): (String, String), +) -> HeaderViolation { + let (line, column) = position(offset, starts); + HeaderViolation { + line, + column, + code, + message, + severity, + remediation, + } +} + +fn parse_fault(starts: &[usize], header: &HeaderSpan, failure: ParseFailure) -> HeaderViolation { + let kind: &str = &header.kind; + violation_at( + starts, + header.start, + (RULE_HEADER_PARSE, HeaderSeverity::Fault), + ( + format!("{kind}() header could not be parsed: {}", failure.message), + failure + .help + .unwrap_or_else(|| format!("Correct the {kind}() header syntax.")), + ), + ) +} + +pub(crate) fn value_violations( + (text, starts): (&[char], &[usize]), + header: &HeaderSpan, + config: HeaderRulesConfig, + parser: &P, +) -> Result, P::Error> { + let header_text: &[char] = slice(text, header.start, header.end); + let kind: &str = &header.kind; + let values: P::Values = match parser.parse(kind, &string(inner_header_text(header_text))) { + ParseOutcome::Parsed(values) => values, + ParseOutcome::Rejected(failure) | ParseOutcome::Raised(_, failure) => { + return Ok(vec![parse_fault(starts, header, failure)]); + } + }; + let mut violations: Vec = Vec::new(); + let described: bool = DESCRIPTION_HEADER_KINDS.contains(&kind); + let span: Option<(usize, usize)> = described + .then(|| description_value_span(header_text)) + .flatten(); + let description: Option> = match (parser.description(&values)?, span) { + (Some(_), Some((start, end))) => Some(decode_description( + slice(header_text, start, end), + header_text[start - 1], + )), + (description, _) => description.map(|description| description.chars().collect()), + }; + if DESCRIPTION_REQUIRED_HEADER_KINDS.contains(&kind) + && description + .as_deref() + .is_none_or(|description| strip(description).is_empty()) + { + violations.push(violation_at( + starts, + header.start, + (RULE_DESCRIPTION_PRESENT, HeaderSeverity::Fault), + ( + format!("{kind}() header requires a description"), + format!( + "Add description \"\" to the {kind}() header; \ + every named resource must be described." + ), + ), + )); + } + if described && let Some(description) = description { + let formatted: Vec = match span { + Some(span) => { + wrap_header_description(&description, header_text, span, config.line_width) + } + None => description, + }; + let limit: usize = config.max_description_lines; + if line_count(&formatted) > limit { + violations.push(violation_at( + starts, + header.start, + (RULE_DESCRIPTION_LENGTH, HeaderSeverity::Fault), + ( + format!("Description exceeds {limit} lines"), + format!("Shorten the description to {limit} lines."), + ), + )); + } + } + Ok(violations) +} + +pub(crate) fn whitespace_violation( + (text, starts): (&[char], &[usize]), + header: &HeaderSpan, +) -> Option { + let header_text: &[char] = slice(text, header.start, header.end); + split_outside_quotes(header_text) + .iter() + .any(|line| rstrip(line).len() != line.len()) + .then(|| { + violation_at( + starts, + header.start, + (RULE_HEADER_WHITESPACE, HeaderSeverity::Warning), + ( + format!("{}() header contains trailing whitespace", header.kind), + "Run sqb format to remove trailing header whitespace.".to_owned(), + ), + ) + }) +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/_helpers/scan.rs b/crates/sqlbuild-rules/src/dsl_headers/_helpers/scan.rs new file mode 100644 index 0000000000..16c9a0567e --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/_helpers/scan.rs @@ -0,0 +1,155 @@ +//! Line-leading parenthesized DSL regions, found past comments and SQL strings. + +use sqlbuild_core::text::main::python_isspace::python_isspace; + +use crate::dsl_headers::_helpers::text::{find, is_word}; +use crate::dsl_headers::constants::{ESCAPE, QUOTES}; +use crate::dsl_headers::models::HeaderSpan; + +/// The end of the SQL string quoted by `quote` that opens at `start`; `''` escapes a quote. +fn sql_string_end(text: &[char], start: usize, quote: char) -> usize { + let mut index: usize = start + 1; + while index < text.len() { + if text[index] == quote { + if text.get(index + 1) == Some("e) { + index += 2; + continue; + } + return index + 1; + } + index += 1; + } + text.len() +} + +/// The end of the comment or string at `index`, if one starts there. +fn skipped_end(text: &[char], index: usize) -> Option { + let next: Option<&char> = text.get(index + 1); + match text[index] { + '-' if next == Some(&'-') => Some( + text[index + 2..] + .iter() + .position(|character| *character == '\n') + .map_or(text.len(), |offset| index + 2 + offset + 1), + ), + '/' if next == Some(&'*') => { + Some(find(text, &['*', '/'], index + 2).map_or(text.len(), |found| found + 2)) + } + quote @ ('\'' | '"') => Some(sql_string_end(text, index, quote)), + _ => None, + } +} + +/// The kind keyword that starts the line at `index`, after spaces, and its offset. +fn leading_kind<'k>(text: &[char], index: usize, kinds: &[&'k str]) -> Option<(&'k str, usize)> { + if index > 0 && text[index - 1] != '\n' { + return None; + } + let mut start: usize = index; + while start < text.len() + && python_isspace(text[start]) + && text[start] != '\r' + && text[start] != '\n' + { + start += 1; + } + kinds.iter().find_map(|kind| { + let keyword: Vec = kind.chars().collect(); + let end: usize = start + keyword.len(); + (text[start..].starts_with(&keyword) && !text.get(end).is_some_and(|next| is_word(*next))) + .then_some((*kind, start)) + }) +} + +/// The span of the parenthesized header body after `body_start`, through an optional `;`. +fn header_end(text: &[char], body_start: usize) -> Option { + let mut index: usize = skip_whitespace(text, body_start); + if text.get(index) != Some(&'(') { + return None; + } + let mut depth: i64 = 0; + let mut in_quote: Option = None; + loop { + let found: usize = (index..text.len()).find(|position| match in_quote { + None => matches!(text[*position], '(' | ')' | '\'' | '"'), + Some(quote) => text[*position] == ESCAPE || text[*position] == quote, + })?; + index = found; + let character: char = text[index]; + if in_quote.is_some() { + if character == ESCAPE { + index += 2; + continue; + } + in_quote = None; + } else if QUOTES.contains(&character) { + in_quote = Some(character); + } else if character == '(' { + depth += 1; + } else { + depth -= 1; + if depth == 0 { + let terminator: usize = skip_whitespace(text, index + 1); + if text.get(terminator) == Some(&';') { + return Some(terminator + 1); + } + return Some(index + 1); + } + } + index += 1; + } +} + +pub(crate) fn skip_whitespace(text: &[char], mut index: usize) -> usize { + while index < text.len() && python_isspace(text[index]) { + index += 1; + } + index +} + +/// The first non-space offset in `start..end`, or `end`. +pub(crate) fn first_non_whitespace(text: &[char], start: usize, end: usize) -> usize { + (start..end) + .find(|index| !python_isspace(text[*index])) + .unwrap_or(end) +} + +/// Line-leading parenthesized regions named by one of `kinds`, sorted by name. +pub(crate) fn delimited_regions( + text: &[char], + kinds: &[&str], + first_only: bool, +) -> Vec { + let mut sorted: Vec<&str> = kinds.to_vec(); + sorted.sort_unstable(); + let mut spans: Vec = Vec::new(); + let mut covered_until: usize = 0; + let mut index: usize = 0; + while index < text.len() { + if let Some(end) = skipped_end(text, index) { + index = end; + continue; + } + let Some((kind, keyword_start)) = leading_kind(text, index, &sorted) else { + index += 1; + continue; + }; + let body_start: usize = keyword_start + kind.chars().count(); + index = body_start; + if keyword_start < covered_until { + continue; + } + if let Some(end) = header_end(text, body_start) { + spans.push(HeaderSpan { + kind: kind.to_owned(), + start: keyword_start, + end, + }); + covered_until = end; + if first_only { + break; + } + } + } + spans +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/_helpers/text.rs b/crates/sqlbuild-rules/src/dsl_headers/_helpers/text.rs new file mode 100644 index 0000000000..5e28fcd25a --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/_helpers/text.rs @@ -0,0 +1,99 @@ +//! Python string operations over character slices, indexed as Python indexes `str`. + +use sqlbuild_core::text::main::python_isalnum::python_isalnum; +use sqlbuild_core::text::main::python_isspace::python_isspace; + +/// `text[start:end]` with Python's clamping; an inverted range is empty. +pub(crate) fn slice(text: &[char], start: usize, end: usize) -> &[char] { + let end: usize = end.min(text.len()); + if start >= end { + return &[]; + } + &text[start..end] +} + +/// `text[start:]` with Python's clamping. +pub(crate) fn tail(text: &[char], start: usize) -> &[char] { + slice(text, start, text.len()) +} + +pub(crate) fn strip(text: &[char]) -> &[char] { + rstrip(lstrip(text)) +} + +pub(crate) fn lstrip(text: &[char]) -> &[char] { + let start: usize = text + .iter() + .position(|character| !python_isspace(*character)) + .unwrap_or(text.len()); + &text[start..] +} + +pub(crate) fn rstrip(text: &[char]) -> &[char] { + let end: usize = text + .iter() + .rposition(|character| !python_isspace(*character)) + .map_or(0, |index| index + 1); + &text[..end] +} + +/// `str.isalpha` for one character: alphanumeric without being a number. +pub(crate) fn is_alpha(character: char) -> bool { + python_isalnum(character) && !character.is_numeric() +} + +/// Python's `\w` for one character. +pub(crate) fn is_word(character: char) -> bool { + character == '_' || python_isalnum(character) +} + +/// `text.find(needle, start)`. +pub(crate) fn find(text: &[char], needle: &[char], start: usize) -> Option { + if needle.is_empty() { + return (start <= text.len()).then_some(start); + } + (start..text.len().saturating_sub(needle.len() - 1)) + .find(|index| text[*index..].starts_with(needle)) +} + +/// `text.rfind(character, 0, end)`. +pub(crate) fn rfind_before(text: &[char], character: char, end: usize) -> Option { + slice(text, 0, end) + .iter() + .rposition(|found| *found == character) +} + +pub(crate) fn contains(text: &[char], needle: &str) -> bool { + let needle: Vec = needle.chars().collect(); + find(text, &needle, 0).is_some() +} + +pub(crate) fn line_count(text: &[char]) -> usize { + text.iter().filter(|character| **character == '\n').count() + 1 +} + +/// The offset where each line of `text` starts. +pub(crate) fn line_starts(text: &[char]) -> Vec { + let mut starts: Vec = vec![0]; + starts.extend( + text.iter() + .enumerate() + .filter(|(_, character)| **character == '\n') + .map(|(index, _)| index + 1), + ); + starts +} + +/// The one-based line and column of `offset`. +pub(crate) fn position(offset: usize, starts: &[usize]) -> (usize, usize) { + let line_index: usize = starts.partition_point(|start| *start <= offset) - 1; + (line_index + 1, offset - starts[line_index] + 1) +} + +pub(crate) fn chars(text: &str) -> Vec { + text.chars().collect() +} + +pub(crate) fn string(text: &[char]) -> String { + text.iter().collect() +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/_helpers/wrap.rs b/crates/sqlbuild-rules/src/dsl_headers/_helpers/wrap.rs new file mode 100644 index 0000000000..9d33c62f64 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/_helpers/wrap.rs @@ -0,0 +1,132 @@ +//! Description prose wrapped as Python's `textwrap.TextWrapper` wraps it, without breaking words. + +use sqlbuild_core::text::main::python_isspace::python_isspace; + +use crate::dsl_headers::_helpers::text::{chars, find, rfind_before, rstrip, slice, strip}; + +fn is_ascii_space(character: char) -> bool { + matches!(character, '\t' | '\n' | '\u{b}' | '\u{c}' | '\r' | ' ') +} + +/// `textwrap` lines of `text` under `width`, the first indented, never breaking words. +fn wrap_lines(text: &[char], width: usize, indent: &[char]) -> Vec> { + let mut chunks: Vec<&[char]> = Vec::new(); + let mut start: usize = 0; + for index in 1..=text.len() { + if index == text.len() || is_ascii_space(text[index]) != is_ascii_space(text[start]) { + chunks.push(&text[start..index]); + start = index; + } + } + chunks.reverse(); + let is_blank = |chunk: &[char]| strip(chunk).is_empty(); + let mut lines: Vec> = Vec::new(); + while !chunks.is_empty() { + let line_indent: &[char] = if lines.is_empty() { indent } else { &[] }; + let available: usize = width.saturating_sub(line_indent.len()); + if chunks.last().is_some_and(|chunk| is_blank(chunk)) && !lines.is_empty() { + chunks.pop(); + } + let mut line: Vec<&[char]> = Vec::new(); + let mut length: usize = 0; + while let Some(chunk) = chunks.last() { + if length + chunk.len() > available { + break; + } + length += chunk.len(); + line.extend(chunks.pop()); + } + if line.is_empty() + && let Some(chunk) = chunks.last() + && chunk.len() > available + { + line.extend(chunks.pop()); + } + if line.last().is_some_and(|chunk| is_blank(chunk)) { + line.pop(); + } + if !line.is_empty() { + let mut rendered: Vec = line_indent.to_vec(); + rendered.extend(line.concat()); + lines.push(rendered); + } + } + lines +} + +/// Offsets of `\n\s*\n` paragraph separators, as `re.split` finds them. +fn paragraphs(text: &[char]) -> Vec<&[char]> { + let mut parts: Vec<&[char]> = Vec::new(); + let mut part_start: usize = 0; + let mut index: usize = 0; + while index < text.len() { + if text[index] == '\n' { + let mut run_end: usize = index + 1; + while run_end < text.len() && python_isspace(text[run_end]) { + run_end += 1; + } + if let Some(last) = text[index + 1..run_end].iter().rposition(|c| *c == '\n') { + parts.push(&text[part_start..index]); + part_start = index + 1 + last + 1; + index = part_start; + continue; + } + } + index += 1; + } + parts.push(&text[part_start..]); + parts +} + +/// Description prose re-wrapped to the widths, keeping blank-line paragraph breaks. +pub(crate) fn wrap_description(description: &[char], first_width: i64, width: i64) -> Vec { + let normalized: Vec = + chars(&description.iter().collect::().replace("\r\n", "\n")); + let normalized: &[char] = strip(&normalized); + let width_columns: usize = usize::try_from(width).unwrap_or(0); + let mut wrapped: Vec> = Vec::new(); + for (paragraph_index, paragraph) in paragraphs(normalized).into_iter().enumerate() { + let words: Vec<&[char]> = paragraph + .split(|character| python_isspace(*character)) + .filter(|word| !word.is_empty()) + .collect(); + let prose: Vec = words.join(&' '); + if prose.is_empty() { + continue; + } + let first: i64 = if paragraph_index == 0 { + first_width + } else { + width + }; + let indent: Vec = vec![' '; usize::try_from(width - first).unwrap_or(0)]; + let mut lines: Vec> = wrap_lines(&prose, width_columns, &indent); + if let Some(line) = lines.first_mut() + && !indent.is_empty() + { + line.drain(..indent.len().min(line.len())); + } + wrapped.push(lines.join(&'\n')); + } + wrapped.join(&['\n', '\n'][..]) +} + +/// A description wrapped to fit the header line holding its value at `span`. +pub(crate) fn wrap_header_description( + description: &[char], + header_text: &[char], + (value_start, value_end): (usize, usize), + line_width: i64, +) -> Vec { + let line_start: usize = + rfind_before(header_text, '\n', value_start).map_or(0, |found| found + 1); + let line_end: usize = find(header_text, &['\n'], value_end).unwrap_or(header_text.len()); + let suffix: i64 = + i64::try_from(rstrip(slice(header_text, value_end, line_end)).len()).unwrap_or(i64::MAX); + let prefix: i64 = i64::try_from(value_start - line_start).unwrap_or(i64::MAX); + wrap_description( + description, + (line_width - prefix - suffix).max(1), + (line_width - suffix).max(1), + ) +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/constants.rs b/crates/sqlbuild-rules/src/dsl_headers/constants.rs new file mode 100644 index 0000000000..0c9142564d --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/constants.rs @@ -0,0 +1,21 @@ +//! DSL header kinds, rule codes and the text the header rules read and write. + +pub const HEADER_KINDS: [&str; 9] = [ + "AUDIT", "CONSTANT", "ENUM", "FUNCTION", "HOOK", "MODEL", "SCENARIO", "SCHEMA", "TEST", +]; +pub const MEASUREMENT_KINDS: [&str; 2] = ["EVIDENCE", "MEASURE"]; +pub(crate) const DESCRIPTION_HEADER_KINDS: [&str; 2] = ["MODEL", "SCENARIO"]; +pub(crate) const DESCRIPTION_REQUIRED_HEADER_KINDS: [&str; 4] = + ["MODEL", "SCENARIO", "FUNCTION", "HOOK"]; +pub(crate) const MODEL_KIND: &str = "MODEL"; +pub const RULE_DESCRIPTION_PRESENT: &str = "description-present"; +pub const RULE_DESCRIPTION_LENGTH: &str = "description-length"; +pub const RULE_LEADING_COMMENT_DESCRIPTION: &str = "leading-comment-description"; +pub const RULE_HEADER_WHITESPACE: &str = "header-whitespace"; +pub const RULE_HEADER_PARSE: &str = "header-parse"; +pub(crate) const DESCRIPTION_KEY: &str = "description"; +pub(crate) const HEADER_INDENT: &str = " "; +pub(crate) const AUDITS_KEYWORD: &str = "audits"; +pub(crate) const AUDIT_WRAP_MAX_PASSES: usize = 64; +pub(crate) const QUOTES: [char; 2] = ['\'', '"']; +pub(crate) const ESCAPE: char = '\\'; diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/format_headers.rs b/crates/sqlbuild-rules/src/dsl_headers/main/format_headers.rs new file mode 100644 index 0000000000..978bad4747 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/main/format_headers.rs @@ -0,0 +1,23 @@ +//! `format_headers`: a file with every header formatting fix applied. + +use crate::dsl_headers::_helpers::formatting::{ + relocate_leading_comment, without_trailing_whitespace, wrap_audits, wrap_descriptions, +}; +use crate::dsl_headers::main::scan_headers::scan_headers; +use crate::dsl_headers::models::{HeaderRulesConfig, HeaderSpan, HeaderViolation}; +use crate::dsl_headers::types::HeaderParser; + +/// `text` with every header formatting fix applied, and the faults relocation reported. +pub fn format_headers( + text: &[char], + config: HeaderRulesConfig, + parser: &P, +) -> Result<(Vec, Vec), P::Error> { + let headers: Vec = scan_headers(text, false); + let trimmed: Vec = without_trailing_whitespace(text, &headers); + let (relocated, faults) = relocate_leading_comment(&trimmed, config, &headers, parser)?; + let relocated_headers: Vec = scan_headers(&relocated, false); + let wrapped: Vec = + wrap_descriptions(&relocated, &relocated_headers, config.line_width, parser)?; + Ok((wrap_audits(&wrapped, config.line_width, parser)?, faults)) +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/inner_header_text.rs b/crates/sqlbuild-rules/src/dsl_headers/main/inner_header_text.rs new file mode 100644 index 0000000000..43e3ca800e --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/main/inner_header_text.rs @@ -0,0 +1,11 @@ +//! `inner_header_text`: the text between a header's parentheses. + +/// The text between a header's first `(` and last `)`, or empty. +pub fn inner_header_text(header_text: &[char]) -> &[char] { + let open: Option = header_text.iter().position(|character| *character == '('); + let close: Option = header_text.iter().rposition(|character| *character == ')'); + match (open, close) { + (Some(open), Some(close)) if close > open => &header_text[open + 1..close], + _ => &[], + } +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/lint_headers.rs b/crates/sqlbuild-rules/src/dsl_headers/main/lint_headers.rs new file mode 100644 index 0000000000..c05f4c115f --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/main/lint_headers.rs @@ -0,0 +1,22 @@ +//! `lint_headers`: every header rule violation of a file's headers. + +use crate::dsl_headers::_helpers::rules::{value_violations, whitespace_violation}; +use crate::dsl_headers::_helpers::text::line_starts; +use crate::dsl_headers::models::{HeaderRulesConfig, HeaderSpan, HeaderViolation}; +use crate::dsl_headers::types::HeaderParser; + +/// Every header rule violation of `headers` in `text`, header by header. +pub fn lint_headers( + text: &[char], + headers: &[HeaderSpan], + config: HeaderRulesConfig, + parser: &P, +) -> Result, P::Error> { + let starts: Vec = line_starts(text); + let mut violations: Vec = Vec::new(); + for header in headers { + violations.extend(value_violations((text, &starts), header, config, parser)?); + violations.extend(whitespace_violation((text, &starts), header)); + } + Ok(violations) +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/measurement_body_ranges.rs b/crates/sqlbuild-rules/src/dsl_headers/main/measurement_body_ranges.rs new file mode 100644 index 0000000000..a576df26f3 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/main/measurement_body_ranges.rs @@ -0,0 +1,24 @@ +//! `measurement_body_ranges`: the query bodies of MEASURE and EVIDENCE blocks. + +use crate::dsl_headers::_helpers::scan::{delimited_regions, first_non_whitespace}; +use crate::dsl_headers::_helpers::text::rfind_before; +use crate::dsl_headers::constants::MEASUREMENT_KINDS; + +/// The authored query ranges inside MEASURE and EVIDENCE blocks. +pub fn measurement_body_ranges(text: &[char]) -> Vec<(usize, usize)> { + let mut ranges: Vec<(usize, usize)> = Vec::new(); + for block in delimited_regions(text, &MEASUREMENT_KINDS, false) { + let opening: usize = (block.start..block.end) + .find(|index| text[*index] == '(') + .unwrap_or(block.start); + let after_opening: usize = opening + 1; + let closing: usize = rfind_before(text, ')', block.end) + .filter(|found| *found >= after_opening) + .unwrap_or(after_opening); + let body_start: usize = first_non_whitespace(text, after_opening, closing); + if body_start < closing { + ranges.push((body_start, closing)); + } + } + ranges +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs b/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs new file mode 100644 index 0000000000..327f9ec2f3 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs @@ -0,0 +1,6 @@ +pub mod format_headers; +pub mod inner_header_text; +pub mod lint_headers; +pub mod measurement_body_ranges; +pub mod scan_headers; +pub mod sql_body_ranges; diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/scan_headers.rs b/crates/sqlbuild-rules/src/dsl_headers/main/scan_headers.rs new file mode 100644 index 0000000000..7fc0afa0ea --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/main/scan_headers.rs @@ -0,0 +1,10 @@ +//! `scan_headers`: every DSL header span of a file. + +use crate::dsl_headers::_helpers::scan::delimited_regions; +use crate::dsl_headers::constants::HEADER_KINDS; +use crate::dsl_headers::models::HeaderSpan; + +/// Every DSL header span in `text`, or only the first. +pub fn scan_headers(text: &[char], first_only: bool) -> Vec { + delimited_regions(text, &HEADER_KINDS, first_only) +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/sql_body_ranges.rs b/crates/sqlbuild-rules/src/dsl_headers/main/sql_body_ranges.rs new file mode 100644 index 0000000000..e1c464c516 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/main/sql_body_ranges.rs @@ -0,0 +1,22 @@ +//! `sql_body_ranges`: the SQL bodies around and between a file's headers. + +use crate::dsl_headers::_helpers::scan::first_non_whitespace; +use crate::dsl_headers::models::HeaderSpan; + +/// Non-empty SQL body ranges around and between the headers. +pub fn sql_body_ranges(text: &[char], headers: &[HeaderSpan]) -> Vec<(usize, usize)> { + let mut ranges: Vec<(usize, usize)> = Vec::new(); + let mut cursor: usize = 0; + for header in headers { + let body_start: usize = first_non_whitespace(text, cursor, header.start); + if body_start < header.start { + ranges.push((body_start, header.start)); + } + cursor = header.end; + } + let tail_start: usize = first_non_whitespace(text, cursor, text.len()); + if tail_start < text.len() { + ranges.push((tail_start, text.len())); + } + ranges +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/mod.rs b/crates/sqlbuild-rules/src/dsl_headers/mod.rs new file mode 100644 index 0000000000..db0bf09b87 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/mod.rs @@ -0,0 +1,10 @@ +//! DSL header regions of SQL files, with their lint rules and formatting fixes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/dsl_headers/models.rs b/crates/sqlbuild-rules/src/dsl_headers/models.rs new file mode 100644 index 0000000000..235d36f011 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/models.rs @@ -0,0 +1,48 @@ +//! Header spans, header rule violations and the parser the rules read header values with. + +/// One located DSL header; offsets are character positions. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HeaderSpan { + pub kind: String, + pub start: usize, + pub end: usize, +} + +/// Whether a header violation blocks formatting. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HeaderSeverity { + Fault, + Warning, +} + +/// One header rule violation at a one-based line and column. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HeaderViolation { + pub line: usize, + pub column: usize, + pub code: &'static str, + pub message: String, + pub severity: HeaderSeverity, + pub remediation: String, +} + +/// The line width and description length the header rules enforce. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HeaderRulesConfig { + pub line_width: i64, + pub max_description_lines: usize, +} + +/// Why a header failed to parse: the error's text and help. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParseFailure { + pub message: String, + pub help: Option, +} + +/// One header parse: values, a header syntax error, or another error the caller raised. +pub enum ParseOutcome { + Parsed(V), + Rejected(ParseFailure), + Raised(E, ParseFailure), +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/tests.rs b/crates/sqlbuild-rules/src/dsl_headers/tests.rs new file mode 100644 index 0000000000..fe047fb91e --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dsl_headers.rs"] +mod dsl_headers; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/dsl_headers/tests/helpers.rs b/crates/sqlbuild-rules/src/dsl_headers/tests/helpers.rs new file mode 100644 index 0000000000..23b2e444fe --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/tests/helpers.rs @@ -0,0 +1,3 @@ +pub(crate) fn chars(text: &str) -> Vec { + text.chars().collect() +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/tests/test_dsl_headers.rs b/crates/sqlbuild-rules/src/dsl_headers/tests/test_dsl_headers.rs new file mode 100644 index 0000000000..cf12f374fe --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/tests/test_dsl_headers.rs @@ -0,0 +1,113 @@ +use crate::dsl_headers::_helpers::quoted::decode_description; +use crate::dsl_headers::_helpers::wrap::wrap_description; +use crate::dsl_headers::main::format_headers::format_headers; +use crate::dsl_headers::main::measurement_body_ranges::measurement_body_ranges; +use crate::dsl_headers::main::scan_headers::scan_headers; +use crate::dsl_headers::main::sql_body_ranges::sql_body_ranges; +use crate::dsl_headers::models::HeaderRulesConfig; +use crate::dsl_headers::tests::helpers::chars; +use crate::dsl_headers::tests::test_types::{ + CompactParser, FormatTestCase, ProseTestCase, ScanTestCase, +}; + +const MODEL_FILE: &str = "-- leading note\n-- second line\nMODEL (\n name orders, \n audits [not_null(column 'order_id'), unique(column 'order_id'), accepted_values(column 'status', values ['a', 'b'])]\n);\nSELECT 1 /* MODEL ( */ , 'MODEL (x)'\n TEST (\"a\\\")\" ) ;\nSELECT 2\n"; + +#[test] +fn given_sql_file_when_scanning_headers_then_matches_python_spans_and_bodies() { + let test_cases = [ + ScanTestCase { + description: "headers past comments and strings, with an escaped quote", + text: MODEL_FILE, + expected_spans: vec![("MODEL", 31, 178), ("TEST", 218, 234)], + expected_bodies: vec![(0, 31), (179, 218), (235, 244)], + expected_measurements: vec![], + }, + ScanTestCase { + description: "a measurement block body and an empty evidence block", + text: "AUDIT (name x);\nMEASURE (\n SELECT 1\n)\nEVIDENCE ( )\n", + expected_spans: vec![("AUDIT", 0, 15)], + expected_bodies: vec![(16, 52)], + expected_measurements: vec![(28, 37)], + }, + ]; + + for test_case in test_cases { + let text: Vec = chars(test_case.text); + let headers = scan_headers(&text, false); + let spans: Vec<(&str, usize, usize)> = headers + .iter() + .map(|header| (header.kind.as_str(), header.start, header.end)) + .collect(); + + assert_eq!(spans, test_case.expected_spans, "{}", test_case.description); + assert_eq!( + sql_body_ranges(&text, &headers), + test_case.expected_bodies, + "{}", + test_case.description + ); + assert_eq!( + measurement_body_ranges(&text), + test_case.expected_measurements, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_model_header_when_formatting_then_matches_python_rewrite() { + let test_cases = [FormatTestCase { + description: "a leading comment moves into the description and long audits explode", + text: MODEL_FILE, + line_width: 60, + expected_text: "MODEL (\n description \"leading note second line\",\n name orders,\n audits [\n not_null(column 'order_id'),\n unique(column 'order_id'),\n accepted_values(column 'status', values ['a', 'b']),\n ]\n);\nSELECT 1 /* MODEL ( */ , 'MODEL (x)'\n TEST (\"a\\\")\" ) ;\nSELECT 2\n", + }]; + + for test_case in test_cases { + let config = HeaderRulesConfig { + line_width: test_case.line_width, + max_description_lines: 10, + }; + let Ok((formatted, _faults)) = + format_headers(&chars(test_case.text), config, &CompactParser); + + assert_eq!( + formatted.iter().collect::(), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_description_text_when_wrapping_and_decoding_then_matches_python() { + let test_cases = [ProseTestCase { + description: "paragraphs, an unbreakable word and every escape form", + prose: " The orders\tmodel joins every customer to supercalifragilisticexpialidociously long words.\n \n Second paragraph here.", + escaped: r#"a\"b\\c\nd\u00e9\U0001F600\uD800\x"#, + expected_wrapped: "The orders model\njoins every customer to\nsupercalifragilisticexpialidociously\nlong words.\n\nSecond paragraph here.", + expected_decoded: "a\"b\\c\nd\u{e9}\u{1f600}\\uD800\\x", + }]; + + for test_case in test_cases { + let wrapped: String = wrap_description(&chars(test_case.prose), 20, 30) + .into_iter() + .collect(); + let decoded: String = decode_description(&chars(test_case.escaped), '"') + .into_iter() + .collect(); + + assert_eq!( + wrapped, test_case.expected_wrapped, + "{}", + test_case.description + ); + assert_eq!( + decoded, test_case.expected_decoded, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/tests/test_types.rs b/crates/sqlbuild-rules/src/dsl_headers/tests/test_types.rs new file mode 100644 index 0000000000..47af0e7211 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/tests/test_types.rs @@ -0,0 +1,51 @@ +use std::convert::Infallible; + +use crate::dsl_headers::models::ParseOutcome; +use crate::dsl_headers::types::HeaderParser; + +/// Parses a header to its text without whitespace or trailing commas. +pub(crate) struct CompactParser; + +impl HeaderParser for CompactParser { + type Values = String; + type Error = Infallible; + + fn parse(&self, _kind: &str, inner: &str) -> ParseOutcome { + let compact: String = inner + .chars() + .filter(|character| !character.is_whitespace()) + .collect(); + ParseOutcome::Parsed(compact.replace(",]", "]").replace(",)", ")")) + } + + fn description(&self, values: &String) -> Result, Infallible> { + Ok(values.contains("description").then(|| values.clone())) + } + + fn same(&self, left: &String, right: &String) -> Result { + Ok(left == right) + } +} + +pub(crate) struct ScanTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) expected_spans: Vec<(&'static str, usize, usize)>, + pub(crate) expected_bodies: Vec<(usize, usize)>, + pub(crate) expected_measurements: Vec<(usize, usize)>, +} + +pub(crate) struct FormatTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) line_width: i64, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct ProseTestCase { + pub(crate) description: &'static str, + pub(crate) prose: &'static str, + pub(crate) escaped: &'static str, + pub(crate) expected_wrapped: &'static str, + pub(crate) expected_decoded: &'static str, +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/types.rs b/crates/sqlbuild-rules/src/dsl_headers/types.rs new file mode 100644 index 0000000000..a8f110de14 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/types.rs @@ -0,0 +1,18 @@ +//! The parser the header rules read header values with. + +use crate::dsl_headers::models::ParseOutcome; + +/// Parses the inner text of a header into values the rules read. +pub trait HeaderParser { + type Values; + type Error; + + /// Parse the text between a header's parentheses as a `kind` header. + fn parse(&self, kind: &str, inner: &str) -> ParseOutcome; + + /// The `description` value when it is a string. + fn description(&self, values: &Self::Values) -> Result, Self::Error>; + + /// Whether two parses hold equal values. + fn same(&self, left: &Self::Values, right: &Self::Values) -> Result; +} diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index 690afcadde..49dca7ea3c 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -4,9 +4,11 @@ pub mod configuration; pub mod constants; +pub mod dsl_headers; pub mod engine; pub mod errors; pub mod models; +pub mod rule_results; pub mod rules; pub mod scope_metadata; pub mod sql_lint; diff --git a/crates/sqlbuild-rules/src/rule_results/_helpers/literals.rs b/crates/sqlbuild-rules/src/rule_results/_helpers/literals.rs new file mode 100644 index 0000000000..b9eecdf2ba --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/_helpers/literals.rs @@ -0,0 +1,70 @@ +//! The long literal a finding points at, and the hint naming every file that repeats it. + +use std::collections::BTreeSet; + +use crate::rule_results::_helpers::paths::{common_folder, owner, posix}; +use crate::rule_results::constants::{ + INHERITED_CONSTANTS, LISTED_PATHS, LOCAL_CONSTANTS, MIN_SCOPED_FOLDER_DEPTH, PROJECT_CONSTANTS, + QUOTE, +}; + +/// The single-quoted literal starting at a one-based position, quotes included. +pub(crate) fn literal_at(lines: &[String], line: i64, column: i64) -> Option { + let (Ok(line), Ok(column)) = (usize::try_from(line), usize::try_from(column)) else { + return None; + }; + let text: Vec = lines.get(line.checked_sub(1)?)?.chars().collect(); + let start: usize = column.checked_sub(1)?; + if text.get(start) != Some("E) { + return None; + } + let mut position: usize = start + 1; + while position < text.len() { + if text[position] == QUOTE { + if text.get(position + 1) == Some("E) { + position += 2; + continue; + } + return Some(text[start..=position].iter().collect()); + } + position += 1; + } + None +} + +/// The hint listing the other files and the folder to declare the shared constant in. +pub(crate) fn hint(others: &[&Vec], every: &BTreeSet>) -> String { + let listed: String = others + .iter() + .take(LISTED_PATHS) + .map(|path| posix(path)) + .collect::>() + .join(", "); + let more: String = if others.len() > LISTED_PATHS { + format!(" and {} more", others.len() - LISTED_PATHS) + } else { + String::new() + }; + let owners: BTreeSet> = every.iter().map(|path| owner(path)).collect(); + let common: Vec = common_folder(&owners.iter().cloned().collect::>()); + let placement: String = if common.len() < MIN_SCOPED_FOLDER_DEPTH { + format!( + "no folder below the project roots holds every use, so declare it once in \ + top-level `{PROJECT_CONSTANTS}`" + ) + } else { + let directory: &str = if owners.len() == 1 && owners.contains(&common) { + LOCAL_CONSTANTS + } else { + INHERITED_CONSTANTS + }; + let folder: String = posix(&common); + format!( + "their nearest common folder is `{folder}`, so declare it once in `{folder}/{directory}`" + ) + }; + format!( + "The identical literal also appears in {listed}{more}; {placement} \ + and reference it with `@const` in each file." + ) +} diff --git a/crates/sqlbuild-rules/src/rule_results/_helpers/mod.rs b/crates/sqlbuild-rules/src/rule_results/_helpers/mod.rs new file mode 100644 index 0000000000..5903be65a5 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod literals; +pub(crate) mod paths; +pub(crate) mod rendering; diff --git a/crates/sqlbuild-rules/src/rule_results/_helpers/paths.rs b/crates/sqlbuild-rules/src/rule_results/_helpers/paths.rs new file mode 100644 index 0000000000..1277793e29 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/_helpers/paths.rs @@ -0,0 +1,40 @@ +//! POSIX path parts compared, parented and joined the way `PurePosixPath` does. + +use crate::rule_results::constants::{CURRENT_DIRECTORY, DECLARATION_GROUP}; + +/// The parts of a relative POSIX path, without empty or `.` segments. +pub(crate) fn parts(path: &str) -> Vec { + path.split('/') + .filter(|part| !part.is_empty() && *part != CURRENT_DIRECTORY) + .map(str::to_owned) + .collect() +} + +/// `PurePosixPath(*parts).as_posix()`. +pub(crate) fn posix(parts: &[String]) -> String { + if parts.is_empty() { + return CURRENT_DIRECTORY.to_owned(); + } + parts.join("/") +} + +/// The folder whose declarations a file sees: above `_sqlbuild/`, else its own folder. +pub(crate) fn owner(path: &[String]) -> Vec { + match path.iter().position(|part| part == DECLARATION_GROUP) { + Some(index) => path[..index].to_vec(), + None => path[..path.len().saturating_sub(1)].to_vec(), + } +} + +/// `os.path.commonpath` of relative folders. +pub(crate) fn common_folder(folders: &[Vec]) -> Vec { + let (Some(lowest), Some(highest)) = (folders.iter().min(), folders.iter().max()) else { + return Vec::new(); + }; + lowest + .iter() + .zip(highest) + .take_while(|(left, right)| left == right) + .map(|(part, _)| part.clone()) + .collect() +} diff --git a/crates/sqlbuild-rules/src/rule_results/_helpers/rendering.rs b/crates/sqlbuild-rules/src/rule_results/_helpers/rendering.rs new file mode 100644 index 0000000000..b6493c8104 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/_helpers/rendering.rs @@ -0,0 +1,108 @@ +//! A rules result as CLI text or as indented, key-sorted JSON. + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +use crate::rule_results::constants::{FIX_AVAILABLE_NOTE, JSON_INDENT}; +use crate::rule_results::models::{ResultCounts, RuleFinding}; + +pub(crate) fn result_text(findings: &[RuleFinding], counts: ResultCounts) -> String { + if findings.is_empty() { + return format!( + "Rules passed: {} models evaluated, 0 findings ({} cache hits, {} misses)", + counts.evaluated_models, counts.cache_hits, counts.cache_misses + ); + } + let mut blocks: Vec = Vec::new(); + for finding in findings { + blocks.push(format!( + "{}:{}:{} [{}] {}\n Remediation: {}", + finding.path, + finding.line, + finding.column, + finding.code, + finding.message, + finding.remediation + )); + if !finding.affected_rules.is_empty() { + blocks.push(format!( + " Affected Rules: {}", + finding.affected_rules.join(", ") + )); + } + if finding.fixable { + blocks.push(format!(" {FIX_AVAILABLE_NOTE}")); + } + } + blocks.push(format!("Found {} Rules findings", findings.len())); + if counts.unevaluated_resources != 0 { + blocks.push(format!( + "{} models evaluated, {} resources could not be evaluated", + counts.evaluated_models, counts.unevaluated_resources + )); + } + blocks.join("\n") +} + +fn integer(value: i64) -> JsonValue { + JsonInteger::parse(&value.to_string()).map_or(JsonValue::Null, JsonValue::Integer) +} + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn finding_json(finding: &RuleFinding) -> JsonValue { + let mut entries: Vec<(String, JsonValue)> = vec![ + ("code".to_owned(), text(&finding.code)), + ("path".to_owned(), text(&finding.path)), + ("line".to_owned(), integer(finding.line)), + ("column".to_owned(), integer(finding.column)), + ("message".to_owned(), text(&finding.message)), + ("remediation".to_owned(), text(&finding.remediation)), + ]; + if !finding.affected_rules.is_empty() { + entries.push(( + "affected_rules".to_owned(), + JsonValue::Array( + finding + .affected_rules + .iter() + .map(|rule| text(rule)) + .collect(), + ), + )); + } + if finding.fixable { + entries.push(("fixable".to_owned(), JsonValue::Bool(true))); + } + JsonValue::Object(entries) +} + +pub(crate) fn result_json(findings: &[RuleFinding], counts: ResultCounts) -> String { + let payload: JsonValue = JsonValue::Object(vec![ + ( + "evaluated_models".to_owned(), + integer(counts.evaluated_models), + ), + ( + "unevaluated_resources".to_owned(), + integer(counts.unevaluated_resources), + ), + ("cache_hits".to_owned(), integer(counts.cache_hits)), + ("cache_misses".to_owned(), integer(counts.cache_misses)), + ( + "finding_count".to_owned(), + integer(i64::try_from(findings.len()).unwrap_or(i64::MAX)), + ), + ( + "findings".to_owned(), + JsonValue::Array(findings.iter().map(finding_json).collect()), + ), + ]); + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(Some(JSON_INDENT)) + }; + dumps(&payload, &JsonDialect::Stdlib(options)).unwrap_or_default() +} diff --git a/crates/sqlbuild-rules/src/rule_results/constants.rs b/crates/sqlbuild-rules/src/rule_results/constants.rs new file mode 100644 index 0000000000..e2bcd24e2a --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/constants.rs @@ -0,0 +1,15 @@ +//! Codes, placement folders and wording of rules results and literal hints. + +pub(crate) const LONG_LITERAL_CODE: &str = "SQBRSQL044"; +pub(crate) const QUOTE: char = '\''; +pub(crate) const LISTED_PATHS: usize = 5; +pub(crate) const MIN_SHARED_FILES: usize = 2; +pub(crate) const MIN_SCOPED_FOLDER_DEPTH: usize = 2; +pub(crate) const DECLARATION_GROUP: &str = "_sqlbuild"; +pub(crate) const LOCAL_CONSTANTS: &str = "_sqlbuild/_constants/"; +pub(crate) const INHERITED_CONSTANTS: &str = "_sqlbuild/constants/"; +pub(crate) const PROJECT_CONSTANTS: &str = "constants/"; +pub(crate) const FIX_AVAILABLE_NOTE: &str = + "Fix available: `sqb format --fix` applies it after compiler verification"; +pub(crate) const JSON_INDENT: usize = 2; +pub(crate) const CURRENT_DIRECTORY: &str = "."; diff --git a/crates/sqlbuild-rules/src/rule_results/main/duplicate_literal_hints.rs b/crates/sqlbuild-rules/src/rule_results/main/duplicate_literal_hints.rs new file mode 100644 index 0000000000..42c0ea2189 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/duplicate_literal_hints.rs @@ -0,0 +1,51 @@ +//! `duplicate_literal_hints`: SQBRSQL044 help naming the other files that repeat a literal. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use crate::rule_results::_helpers::literals::{hint, literal_at}; +use crate::rule_results::_helpers::paths::parts; +use crate::rule_results::constants::{LONG_LITERAL_CODE, MIN_SHARED_FILES}; +use crate::rule_results::models::RuleFinding; + +/// Each finding's remediation prefixed with its shared-literal hint, when other files repeat it. +pub fn duplicate_literal_hints( + findings: &[RuleFinding], + mut lines: impl FnMut(&str) -> Result, E>, +) -> Result>, E> { + let mut texts: HashMap> = HashMap::new(); + let mut literals: BTreeMap = BTreeMap::new(); + for (index, finding) in findings.iter().enumerate() { + if finding.code != LONG_LITERAL_CODE || finding.unevaluated { + continue; + } + if !texts.contains_key(&finding.path) { + texts.insert(finding.path.clone(), lines(&finding.path)?); + } + let file_lines: &[String] = texts.get(&finding.path).map_or(&[], Vec::as_slice); + if let Some(literal) = literal_at(file_lines, finding.line, finding.column) { + literals.insert(index, literal); + } + } + let mut paths_by_literal: HashMap<&str, BTreeSet>> = HashMap::new(); + for (index, literal) in &literals { + paths_by_literal + .entry(literal) + .or_default() + .insert(parts(&findings[*index].path)); + } + let mut hinted: Vec> = vec![None; findings.len()]; + for (index, literal) in &literals { + let every: &BTreeSet> = &paths_by_literal[literal.as_str()]; + if every.len() < MIN_SHARED_FILES { + continue; + } + let own: Vec = parts(&findings[*index].path); + let others: Vec<&Vec> = every.iter().filter(|path| **path != own).collect(); + hinted[*index] = Some(format!( + "{} {}", + hint(&others, every), + findings[*index].remediation + )); + } + Ok(hinted) +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/format_result.rs b/crates/sqlbuild-rules/src/rule_results/main/format_result.rs new file mode 100644 index 0000000000..02f3ace415 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/format_result.rs @@ -0,0 +1,12 @@ +//! `format_result`: one rules result for the requested CLI surface. + +use crate::rule_results::_helpers::rendering::{result_json, result_text}; +use crate::rule_results::models::{ResultCounts, RuleFinding}; + +/// The result as indented JSON, or as the text report. +pub fn format_result(findings: &[RuleFinding], counts: ResultCounts, json_output: bool) -> String { + if json_output { + return result_json(findings, counts); + } + result_text(findings, counts) +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/mod.rs b/crates/sqlbuild-rules/src/rule_results/main/mod.rs new file mode 100644 index 0000000000..b7461bbc4e --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/mod.rs @@ -0,0 +1,2 @@ +pub mod duplicate_literal_hints; +pub mod format_result; diff --git a/crates/sqlbuild-rules/src/rule_results/mod.rs b/crates/sqlbuild-rules/src/rule_results/mod.rs new file mode 100644 index 0000000000..c4c8014e78 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/mod.rs @@ -0,0 +1,9 @@ +//! Rules results rendered for the CLI, and repeated long literals pointed at a shared constant. + +pub(crate) mod _helpers; +mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/rule_results/models.rs b/crates/sqlbuild-rules/src/rule_results/models.rs new file mode 100644 index 0000000000..d98ab5b21c --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/models.rs @@ -0,0 +1,24 @@ +//! One rules finding and the counts a rules result reports. + +/// One rules finding; `path` is the project-relative POSIX path. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RuleFinding { + pub code: String, + pub path: String, + pub line: i64, + pub column: i64, + pub message: String, + pub remediation: String, + pub unevaluated: bool, + pub affected_rules: Vec, + pub fixable: bool, +} + +/// The model, resource and cache counts of one rules result. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ResultCounts { + pub evaluated_models: i64, + pub unevaluated_resources: i64, + pub cache_hits: i64, + pub cache_misses: i64, +} diff --git a/crates/sqlbuild-rules/src/rule_results/tests.rs b/crates/sqlbuild-rules/src/rule_results/tests.rs new file mode 100644 index 0000000000..d63fc626d1 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_rule_results.rs"] +mod rule_results; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs b/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs new file mode 100644 index 0000000000..cc70c1672c --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs @@ -0,0 +1,126 @@ +use std::convert::Infallible; + +use crate::rule_results::main::duplicate_literal_hints::duplicate_literal_hints; +use crate::rule_results::main::format_result::format_result; +use crate::rule_results::models::{ResultCounts, RuleFinding}; +use crate::rule_results::tests::test_types::{HintTestCase, ResultTestCase}; + +const SUFFIX: &str = " and reference it with `@const` in each file. R."; + +#[test] +fn given_repeated_long_literals_when_hinting_then_matches_python_wording() { + let test_cases = [ + HintTestCase { + description: "two files in one folder share a local constants folder", + paths: vec!["models/a/x.sql", "models/a/y.sql"], + expected_hint: Some( + "The identical literal also appears in models/a/y.sql; their nearest common folder is `models/a`, so declare it once in `models/a/_sqlbuild/_constants/`", + ), + }, + HintTestCase { + description: "a declaration group file and subfolders inherit constants", + paths: vec![ + "models/a/_sqlbuild/x.sql", + "models/a/b/y.sql", + "models/a/c/z.sql", + ], + expected_hint: Some( + "The identical literal also appears in models/a/b/y.sql, models/a/c/z.sql; their nearest common folder is `models/a`, so declare it once in `models/a/_sqlbuild/constants/`", + ), + }, + HintTestCase { + description: "paths sort by parts and only five are listed", + paths: vec![ + "models/a-b/x.sql", + "models/a/b/x.sql", + "models/a/y.sql", + "m/1.sql", + "m/2.sql", + "m/3.sql", + "m/4.sql", + ], + expected_hint: Some( + "The identical literal also appears in m/1.sql, m/2.sql, m/3.sql, m/4.sql, models/a/b/x.sql and 1 more; no folder below the project roots holds every use, so declare it once in top-level `constants/`", + ), + }, + HintTestCase { + description: "a literal in one file gets no hint", + paths: vec!["models/a/x.sql"], + expected_hint: None, + }, + ]; + + for test_case in test_cases { + let findings: Vec = test_case + .paths + .iter() + .map(|path| RuleFinding { + code: "SQBRSQL044".to_owned(), + path: (*path).to_owned(), + line: 1, + column: 8, + remediation: "R.".to_owned(), + ..RuleFinding::default() + }) + .collect(); + let Ok(hinted) = duplicate_literal_hints(&findings, |_| { + Ok::, Infallible>(vec!["SELECT '\u{e9} long literal'".to_owned()]) + }); + + assert_eq!( + hinted[0], + test_case + .expected_hint + .map(|hint| format!("{hint}{SUFFIX}")), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rules_result_when_formatting_then_matches_python_text_and_json() { + let test_cases = [ + ResultTestCase { + description: "text lists affected rules, the fix note and unevaluated resources", + json_output: false, + expected_text: "models/o.sql:3:4 [SQBRSQL001] bad \u{e9}\n Remediation: Fix it.\n Affected Rules: A, B\n Fix available: `sqb format --fix` applies it after compiler verification\nFound 1 Rules findings\n2 models evaluated, 1 resources could not be evaluated", + }, + ResultTestCase { + description: "JSON is indented, key-sorted and ASCII-escaped", + json_output: true, + expected_text: "{\n \"cache_hits\": 1,\n \"cache_misses\": 1,\n \"evaluated_models\": 2,\n \"finding_count\": 1,\n \"findings\": [\n {\n \"affected_rules\": [\n \"A\",\n \"B\"\n ],\n \"code\": \"SQBRSQL001\",\n \"column\": 4,\n \"fixable\": true,\n \"line\": 3,\n \"message\": \"bad \\u00e9\",\n \"path\": \"models/o.sql\",\n \"remediation\": \"Fix it.\"\n }\n ],\n \"unevaluated_resources\": 1\n}", + }, + ]; + let finding = RuleFinding { + code: "SQBRSQL001".to_owned(), + path: "models/o.sql".to_owned(), + line: 3, + column: 4, + message: "bad \u{e9}".to_owned(), + remediation: "Fix it.".to_owned(), + unevaluated: false, + affected_rules: vec!["A".to_owned(), "B".to_owned()], + fixable: true, + }; + let counts = ResultCounts { + evaluated_models: 2, + unevaluated_resources: 1, + cache_hits: 1, + cache_misses: 1, + }; + + for test_case in test_cases { + let rendered: String = format_result( + std::slice::from_ref(&finding), + counts, + test_case.json_output, + ); + + assert_eq!( + rendered, test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs b/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs new file mode 100644 index 0000000000..97f61de3cc --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs @@ -0,0 +1,11 @@ +pub(crate) struct HintTestCase { + pub(crate) description: &'static str, + pub(crate) paths: Vec<&'static str>, + pub(crate) expected_hint: Option<&'static str>, +} + +pub(crate) struct ResultTestCase { + pub(crate) description: &'static str, + pub(crate) json_output: bool, + pub(crate) expected_text: &'static str, +} diff --git a/crates/sqlbuild-scopes/src/lib.rs b/crates/sqlbuild-scopes/src/lib.rs index 8fdcac1590..21704797df 100644 --- a/crates/sqlbuild-scopes/src/lib.rs +++ b/crates/sqlbuild-scopes/src/lib.rs @@ -8,3 +8,4 @@ pub mod relationship_names; pub mod scope_cache; pub mod scope_index; pub mod scope_report; +pub mod scope_text; diff --git a/crates/sqlbuild-scopes/src/scope_text/_helpers/declarations.rs b/crates/sqlbuild-scopes/src/scope_text/_helpers/declarations.rs new file mode 100644 index 0000000000..84c1cc32b2 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/_helpers/declarations.rs @@ -0,0 +1,326 @@ +//! Declaration rows, their details, explanations and move previews. + +use std::collections::HashSet; + +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::json::models::JsonValue; + +use crate::scope_report::models::{DeclarationReport, MovePreview}; +use crate::scope_text::_helpers::layout::{ + connector, display_path, field, header, none_line, plain_header, scope_label, +}; +use crate::scope_text::constants::{NONE_TEXT, PATH_NONE, UNUSED_MARKER, USED_MARKER}; +use crate::scope_text::models::ScopeTextRequest; +use crate::scope_text::types::Styler; + +/// How one block of declaration rows is styled and how much it shows. +pub(crate) struct DeclarationStyle<'a> { + pub(crate) used: &'a HashSet, + pub(crate) used_marker: Option>, + pub(crate) name: Option>, + /// Whether to show details; `None` follows `--verbose`. + pub(crate) detailed: Option, +} + +impl<'a> DeclarationStyle<'a> { + /// Rows marking `used` declarations, in the default styles. + pub(crate) fn marking(used: &'a HashSet) -> Self { + Self { + used, + used_marker: None, + name: None, + detailed: None, + } + } +} + +fn integer_text(value: &JsonValue) -> Option { + match value { + JsonValue::Integer(integer) => Some(integer.as_str().to_owned()), + JsonValue::Bool(flag) => Some(if *flag { "True" } else { "False" }.to_owned()), + _ => None, + } +} + +fn string_text(value: &JsonValue) -> Option<&str> { + match value { + JsonValue::String(text) => Some(text), + _ => None, + } +} + +/// The metadata facts a detailed row lists. +fn metadata_parts(declaration: &DeclarationReport) -> Vec { + let value = |key: &str| -> Option<&JsonValue> { + declaration + .metadata + .iter() + .rev() + .find(|(name, _)| name == key) + .map(|(_, value)| value) + }; + let mut parts: Vec = Vec::new(); + if let Some(JsonValue::Array(parameters)) = value("parameters") { + parts.push(format!("params {}", parameters.len())); + } + if let Some(members) = value("member_count").and_then(integer_text) { + parts.push(format!("members {members}")); + } + if let Some(scalar_type) = value("scalar_type").and_then(string_text) { + parts.push(format!("type {scalar_type}")); + } + if let Some(logical_type) = value("logical_type").and_then(string_text) { + parts.push(format!("type {logical_type}")); + } + if let Some(collection) = value("collection_kind").and_then(string_text) { + parts.push(match value("item_count").and_then(integer_text) { + Some(count) => format!("{collection} {count}"), + None => collection.to_owned(), + }); + } + if let Some(role_root) = value("role_root").and_then(string_text) { + parts.push(format!("role {role_root}")); + } + if let Some(bucket_path) = value("bucket_path").and_then(string_text) { + parts.push(format!("bucket {bucket_path} (navigation only)")); + } + parts +} + +fn declaration_details(declaration: &DeclarationReport) -> Vec { + let mut details: Vec = vec![declaration.kind.clone(), scope_label(&declaration.scope)]; + if let Some(visibility) = &declaration.visibility { + details.push(match &visibility.through { + Some(through) => format!("{} through {through}", visibility.reason), + None => visibility.reason.clone(), + }); + } + if let Some(reason) = &declaration.inaccessible_reason { + details.push(reason.clone()); + } + details.extend(metadata_parts(declaration)); + details +} + +/// One row per declaration with its used marker, details and definition location. +pub(crate) fn declaration_lines( + (style, request): (CliStyle, &ScopeTextRequest), + declarations: &[DeclarationReport], + options: &DeclarationStyle<'_>, +) -> Vec { + if declarations.is_empty() { + return vec![none_line(style)]; + } + let show_details: bool = options.detailed.unwrap_or(request.verbose); + let mut lines: Vec = Vec::new(); + for (index, declaration) in declarations.iter().enumerate() { + let marker: String = if options.used.contains(&declaration.identity) { + options + .used_marker + .map_or_else(|| style.success(USED_MARKER), |styler| styler(USED_MARKER)) + } else { + style.muted(UNUSED_MARKER) + }; + let name: String = options.name.map_or_else( + || style.object_name(&declaration.identity), + |styler| styler(&declaration.identity), + ); + let mut line: String = format!( + " {} {marker} {name}", + connector(style, index == declarations.len() - 1) + ); + if show_details { + let details: String = declaration_details(declaration).join("; "); + line.push_str(&format!(" {}", style.muted(&format!("[{details}]")))); + } + if request.paths != PATH_NONE { + let location = &declaration.definition; + let position: String = if show_details { + format!("{}:{}", location.line, location.column) + } else { + location.line.to_string() + }; + let path: String = display_path(Some(&location.path), &request.paths); + line.push_str(&format!(" {}", style.muted(&format!("{path}:{position}")))); + } + lines.push(line); + } + lines +} + +/// One declaration group of a move preview and how it is styled. +struct MoveSection<'a> { + title: &'a str, + values: &'a [DeclarationReport], + optional: bool, + warned: bool, + used_marker: Option>, + name: Option>, +} + +impl<'a> MoveSection<'a> { + fn plain(title: &'a str, values: &'a [DeclarationReport], optional: bool) -> Self { + Self { + title, + values, + optional, + warned: false, + used_marker: None, + name: None, + } + } +} + +fn joined_or_none(style: CliStyle, values: &[String]) -> String { + if values.is_empty() { + return style.muted(NONE_TEXT); + } + values.join(", ") +} + +fn text_or_none(style: CliStyle, value: Option<&str>) -> String { + value.map_or_else(|| style.muted(NONE_TEXT), str::to_owned) +} + +/// The explained declaration's detailed row followed by its ownership facts. +pub(crate) fn explanation_lines( + (style, request): (CliStyle, &ScopeTextRequest), + declaration: &DeclarationReport, + used: &HashSet, +) -> Vec { + let mut lines: Vec = declaration_lines( + (style, request), + std::slice::from_ref(declaration), + &DeclarationStyle { + detailed: Some(true), + ..DeclarationStyle::marking(used) + }, + ); + let required_scope: Option = declaration.required_scope.as_deref().map(scope_label); + let facts: [(&str, String); 8] = [ + ("Owner", text_or_none(style, declaration.owner.as_deref())), + ( + "Owning path", + text_or_none(style, declaration.owning_path.as_deref()), + ), + ("Consumers", joined_or_none(style, &declaration.consumers)), + ( + "Dependencies", + joined_or_none(style, &declaration.dependencies), + ), + ("Grants", joined_or_none(style, &declaration.grants)), + ( + "Required scope", + text_or_none(style, required_scope.as_deref()), + ), + ( + "Required path", + text_or_none(style, declaration.required_path.as_deref()), + ), + ( + "Promotion impact", + joined_or_none(style, &declaration.promotion_impact), + ), + ]; + for (label, rendered) in facts { + lines.push(format!( + " {} {rendered}", + style.muted(&format!("{label}:")) + )); + } + lines +} + +/// The move preview: destination facts, then retained, gained and lost declarations. +pub(crate) fn move_lines( + (style, request): (CliStyle, &ScopeTextRequest), + preview: &MovePreview, + used: &HashSet, +) -> Vec { + let mut lines: Vec = vec![ + style.section("Move preview"), + field(style, "Resource", &preview.resource), + field( + style, + "Destination", + &display_path(Some(&preview.destination), &request.paths), + ), + field( + style, + "Ownership root", + &display_path(preview.new_ownership_root.as_deref(), &request.paths), + ), + ]; + let lost_warning: bool = !preview.lost.is_empty(); + let warning_strong = |text: &str| style.warning_strong(text); + let warning = |text: &str| style.warning(text); + let error = |text: &str| style.error(text); + let success = |text: &str| style.success(text); + let sections: [MoveSection<'_>; 5] = [ + MoveSection::plain("Retained", &preview.retained, false), + MoveSection { + name: Some(&success), + ..MoveSection::plain("Gained", &preview.gained, false) + }, + MoveSection { + warned: lost_warning, + used_marker: Some(&error), + ..MoveSection::plain("Lost", &preview.lost, false) + }, + MoveSection::plain("Private retained", &preview.private_retained, true), + MoveSection::plain( + "Relationship retained", + &preview.relationship_retained, + true, + ), + ]; + for MoveSection { + title, + values, + optional, + warned, + used_marker, + name, + } in sections + { + if optional && !request.verbose && values.is_empty() { + continue; + } + let count: String = format!("({})", values.len()); + let section_header: String = if warned { + header(title, &count, (&warning_strong, &warning)) + } else { + plain_header(style, title, &count) + }; + lines.push(format!(" {section_header}")); + let rows: Vec = declaration_lines( + (style, request), + values, + &DeclarationStyle { + used_marker, + name, + ..DeclarationStyle::marking(used) + }, + ); + lines.extend(rows.into_iter().map(|row| format!(" {row}"))); + } + let invalidated: &[String] = &preview.invalidated_usages; + let count: String = format!("({})", invalidated.len()); + let invalidated_header: String = if invalidated.is_empty() { + plain_header(style, "Invalidated usages", &count) + } else { + header( + "Invalidated usages", + &count, + (&|text: &str| style.error_strong(text), &error), + ) + }; + lines.push(format!(" {invalidated_header}")); + for identity in invalidated { + lines.push(format!(" {}", style.error(&format!("- {identity}")))); + } + if invalidated.is_empty() { + lines.push(format!(" {}", style.muted(NONE_TEXT))); + } + lines +} diff --git a/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs b/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs new file mode 100644 index 0000000000..4541c5dc2c --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs @@ -0,0 +1,265 @@ +//! Fields, headers, connectors, paths and follow-up commands of scope text output. + +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::scope_report::models::{DiagnosticFacts, ReportDiagnostic, ScopeSection}; +use crate::scope_text::constants::{ + BRANCH_CONNECTOR, CURRENT_DIRECTORY, DEFAULT_PAGE_SIZE, ERROR_SEVERITY, GLOBAL_ALL, + GLOBAL_SUMMARY, LAST_CONNECTOR, NONE_TEXT, PATH_COMPACT, PATH_NONE, PATH_RELATIVE, + SCOPE_LABELS, +}; +use crate::scope_text::models::ScopeTextRequest; +use crate::scope_text::types::Styler; + +/// ` Label: value`. +pub(crate) fn field(style: CliStyle, label: &str, value: &str) -> String { + format!(" {} {value}", style.muted(&format!("{label}:"))) +} + +/// A section title with its count, each in its own style. +pub(crate) fn header( + title: &str, + count: &str, + (title_style, count_style): (Styler<'_>, Styler<'_>), +) -> String { + format!("{} {}", title_style(title), count_style(count)) +} + +/// A section title in the section style with a muted count. +pub(crate) fn plain_header(style: CliStyle, title: &str, count: &str) -> String { + header( + title, + count, + (&|text: &str| style.section(text), &|text: &str| { + style.muted(text) + }), + ) +} + +pub(crate) fn connector(style: CliStyle, last: bool) -> String { + style.muted(if last { + LAST_CONNECTOR + } else { + BRANCH_CONNECTOR + }) +} + +pub(crate) fn none_line(style: CliStyle) -> String { + format!(" {}", style.muted(NONE_TEXT)) +} + +/// The label a scope kind reads as. +pub(crate) fn scope_label(scope: &str) -> String { + SCOPE_LABELS + .iter() + .find(|(kind, _)| *kind == scope) + .map_or_else(|| scope.to_owned(), |(_, label)| (*label).to_owned()) +} + +/// A path as the `--paths` mode shows it. +pub(crate) fn display_path(value: Option<&str>, mode: &str) -> String { + let Some(value) = value else { + return NONE_TEXT.to_owned(); + }; + if mode == PATH_NONE { + return "(hidden)".to_owned(); + } + if mode == PATH_COMPACT { + return posix_name(value).to_owned(); + } + value.to_owned() +} + +/// `PurePosixPath(value).name`. +fn posix_name(value: &str) -> &str { + let trimmed: &str = value.trim_end_matches('/'); + let name: &str = trimmed.rsplit('/').next().unwrap_or(trimmed); + if name == CURRENT_DIRECTORY { "" } else { name } +} + +/// `()`, or `( of [, collapsed])` for a paged section. +pub(crate) fn section_count(section: Option<&ScopeSection>, count: usize) -> String { + let Some(section) = section else { + return format!("({count})"); + }; + let suffix: String = if section.collapsed_count == 0 { + String::new() + } else { + format!(", {} collapsed", section.collapsed_count) + }; + format!("({} of {}{suffix})", section.returned, section.total) +} + +/// `count singular`, or the plural when the count is not one. +pub(crate) fn count_noun(count: i64, singular: &str, plural: &str) -> String { + format!("{count} {}", if count == 1 { singular } else { plural }) +} + +pub(crate) fn completeness_line(style: CliStyle, complete: bool) -> String { + let state: String = if complete { + style.success("complete") + } else { + style.warning("partial") + }; + format!("{} {state}", style.muted("Completeness:")) +} + +fn diagnostic_facts(diagnostics: &[ReportDiagnostic]) -> Vec<&DiagnosticFacts> { + diagnostics + .iter() + .filter_map(|diagnostic| match diagnostic { + ReportDiagnostic::Raised(facts) => Some(facts), + ReportDiagnostic::Index(_) => None, + }) + .collect() +} + +/// Whether any diagnostic is an error. +pub(crate) fn has_error(diagnostics: &[ReportDiagnostic]) -> bool { + diagnostic_facts(diagnostics) + .iter() + .any(|facts| facts.severity == ERROR_SEVERITY) +} + +/// The diagnostics section; empty unless there are diagnostics or the output is verbose. +pub(crate) fn diagnostic_lines( + style: CliStyle, + diagnostics: &[ReportDiagnostic], + verbose: bool, +) -> Vec { + let facts: Vec<&DiagnosticFacts> = diagnostic_facts(diagnostics); + if facts.is_empty() && !verbose { + return Vec::new(); + } + let count: String = format!("({})", facts.len()); + let mut lines: Vec = vec![if facts.is_empty() { + plain_header(style, "Diagnostics", &count) + } else { + header( + "Diagnostics", + &count, + (&|text: &str| style.error_strong(text), &|text: &str| { + style.error(text) + }), + ) + }]; + if facts.is_empty() { + lines.push(none_line(style)); + } + for diagnostic in facts { + let severity: String = diagnostic.severity.to_uppercase(); + let severity_text: String = if diagnostic.severity == ERROR_SEVERITY { + style.error_strong(&severity) + } else { + style.warning_strong(&severity) + }; + let location: String = diagnostic + .path + .as_ref() + .map(|path| format!(" {}", style.muted(path))) + .unwrap_or_default(); + lines.push(format!( + " {severity_text} {}{location}: {}", + diagnostic.code, diagnostic.message + )); + } + lines +} + +/// Python's `shlex.quote`. +fn shell_quote(text: &str) -> String { + if text.is_empty() { + return "''".to_owned(); + } + let safe: bool = text + .chars() + .all(|character| character.is_ascii_alphanumeric() || "_@%+=:,./-".contains(character)); + if safe { + return text.to_owned(); + } + format!("'{}'", text.replace('\'', "'\"'\"'")) +} + +/// What a follow-up command adds to the request's own options. +#[derive(Clone, Copy, Default)] +pub(crate) struct FollowUp<'a> { + pub(crate) browse: Option<&'a str>, + pub(crate) list_path: Option<&'a str>, + pub(crate) after: Option<&'a str>, + pub(crate) globals_all: bool, +} + +fn option(name: &str, value: &str) -> [String; 2] { + [name.to_owned(), value.to_owned()] +} + +/// The `sqb scope` command that repeats this request with `extra`, shell-quoted. +pub(crate) fn follow_up(request: &ScopeTextRequest, extra: FollowUp<'_>) -> String { + let mut args: Vec = vec!["sqb".to_owned(), "scope".to_owned()]; + match &request.target { + Some(target) => args.push(target.clone()), + None => args.extend(option("--at", request.at.as_deref().unwrap_or_default())), + } + if let Some(browse) = extra.browse { + args.extend(option("--browse", browse)); + } + if let Some(list_path) = extra.list_path { + args.extend(option("--list", list_path)); + } + if let Some(defined_under) = &request.defined_under { + args.extend(option("--defined-under", defined_under)); + } + for kind in &request.kinds { + args.extend(option("--kind", kind)); + } + if let Some(glob) = &request.glob { + args.extend(option("--match", glob)); + } + if request.used_only { + args.push("--used-only".to_owned()); + } + if extra.globals_all { + if let Some(as_path) = &request.as_path { + args.extend(option("--as-path", as_path)); + } + if request.include_nearby { + args.push("--include-nearby".to_owned()); + } + if request.nearby_depth != 1 { + args.extend(option("--nearby-depth", &request.nearby_depth.to_string())); + } + if request.dependency_depth != 0 { + args.extend(option( + "--dependency-depth", + &request.dependency_depth.to_string(), + )); + } + if let Some(explain) = &request.explain { + args.extend(option("--explain", explain)); + } + } + if request.page_size != DEFAULT_PAGE_SIZE { + args.extend(option("--page-size", &request.page_size.to_string())); + } + if let Some(after) = extra.after { + args.extend(option("--after", after)); + } + if extra.globals_all { + args.extend(option("--globals", GLOBAL_ALL)); + } else if request.globals != GLOBAL_SUMMARY { + args.extend(option("--globals", &request.globals)); + } + if request.paths != PATH_RELATIVE { + args.extend(option("--paths", &request.paths)); + } + if request.verbose { + args.push("--verbose".to_owned()); + } + if request.no_cache { + args.push("--no-cache".to_owned()); + } + args.iter() + .map(|arg| shell_quote(arg)) + .collect::>() + .join(" ") +} diff --git a/crates/sqlbuild-scopes/src/scope_text/_helpers/mod.rs b/crates/sqlbuild-scopes/src/scope_text/_helpers/mod.rs new file mode 100644 index 0000000000..ec69bbd5b3 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod declarations; +pub(crate) mod layout; +pub(crate) mod results; diff --git a/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs b/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs new file mode 100644 index 0000000000..ea40a71473 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs @@ -0,0 +1,304 @@ +//! The text of one scope report, folder browse or declaration list. + +use std::collections::HashSet; + +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::scope_report::models::{ + BrowseResult, DeclarationReport, ListResult, ScopeReport, ScopeSection, +}; +use crate::scope_text::_helpers::declarations::{ + DeclarationStyle, declaration_lines, explanation_lines, move_lines, +}; +use crate::scope_text::_helpers::layout::{ + FollowUp, completeness_line, connector, count_noun, diagnostic_lines, display_path, field, + follow_up, none_line, plain_header, scope_label, section_count, +}; +use crate::scope_text::constants::{LEGEND_MARKERS, LEGEND_VERBOSE_HINT}; +use crate::scope_text::models::ScopeTextRequest; + +fn joined(lines: Vec) -> String { + format!("{}\n", lines.join("\n")) +} + +fn report_section<'a>(report: &'a ScopeReport, name: &str) -> Option<&'a ScopeSection> { + report.sections.iter().find(|section| section.name == name) +} + +fn header_lines( + (style, request): (CliStyle, &ScopeTextRequest), + report: &ScopeReport, +) -> Vec { + let resource = &report.resource; + let mut lines: Vec = vec![ + style.section("Scope"), + field(style, "Target", &resource.target), + ]; + if let Some(identity) = &resource.identity { + lines.push(field(style, "Resource", identity)); + } + if let Some(path) = &resource.path { + lines.push(field( + style, + "Path", + &display_path(Some(path), &request.paths), + )); + } + let mut labels: Vec = Vec::new(); + if resource.prospective { + labels.push("prospective".to_owned()); + } + if resource.directory { + labels.push("directory".to_owned()); + } + if resource.duplicate_count != 0 { + labels.push(count_noun(resource.duplicate_count, "match", "matches")); + } + if !labels.is_empty() { + lines.push(field(style, "Status", &labels.join(", "))); + } + lines +} + +fn chain_lines( + (style, request): (CliStyle, &ScopeTextRequest), + report: &ScopeReport, +) -> Vec { + let mut lines: Vec = vec![String::new(), style.section("Scope chain")]; + if report.scope_chain.is_empty() { + lines.push(none_line(style)); + } + for (index, entry) in report.scope_chain.iter().enumerate() { + let path: String = display_path(Some(&entry.path), &request.paths); + lines.push(format!( + " {} {} {}", + connector(style, index == report.scope_chain.len() - 1), + scope_label(&entry.kind), + style.muted(&format!("{path} ({})", entry.declaration_count)) + )); + } + lines +} + +fn availability_lines( + (style, request): (CliStyle, &ScopeTextRequest), + report: &ScopeReport, + used: &HashSet, +) -> Vec { + let groups: [(&str, &[DeclarationReport], &str, bool); 3] = [ + ("Available", &report.available, "available", false), + ( + "Relationship grants", + &report.relationship_scope, + "relationship_scope", + true, + ), + ( + "Nearby unavailable", + &report.nearby_unavailable, + "nearby_unavailable", + true, + ), + ]; + let mut lines: Vec = Vec::new(); + for (title, declarations, name, optional) in groups { + let section: Option<&ScopeSection> = report_section(report, name); + let collapsed_count: i64 = section.map_or(0, |section| section.collapsed_count); + if optional && !request.verbose && declarations.is_empty() && collapsed_count == 0 { + continue; + } + lines.push(String::new()); + lines.push(plain_header( + style, + title, + §ion_count(section, declarations.len()), + )); + lines.extend(declaration_lines( + (style, request), + declarations, + &DeclarationStyle::marking(used), + )); + if collapsed_count != 0 { + let globals: String = count_noun(collapsed_count, "global", "globals"); + let command: String = follow_up( + request, + FollowUp { + globals_all: true, + ..FollowUp::default() + }, + ); + lines.push(format!( + " {}", + style.muted(&format!("\u{2026} {globals} collapsed; run {command}")) + )); + } + } + lines +} + +/// The report of one target: used, scope chain, available declarations and previews. +pub(crate) fn report_text( + (style, request): (CliStyle, &ScopeTextRequest), + report: &ScopeReport, +) -> String { + let mut lines: Vec = header_lines((style, request), report); + let used: HashSet = report + .used + .iter() + .map(|item| item.identity.clone()) + .collect(); + lines.push(String::new()); + lines.push(plain_header( + style, + "Used", + &format!("({})", report.used.len()), + )); + lines.extend(declaration_lines( + (style, request), + &report.used, + &DeclarationStyle::marking(&used), + )); + lines.extend(chain_lines((style, request), report)); + lines.extend(availability_lines((style, request), report, &used)); + if let Some(explanation) = &report.explanation { + lines.push(String::new()); + lines.push(style.section("Explanation")); + match &explanation.declaration { + None => lines.push(format!(" {}", style.muted("(declaration not resolved)"))), + Some(declaration) => { + lines.extend(explanation_lines((style, request), declaration, &used)); + } + } + } + if let Some(preview) = &report.move_preview { + lines.push(String::new()); + lines.extend(move_lines((style, request), preview, &used)); + } + if !request.verbose { + lines.push(String::new()); + lines.push(style.muted(&format!("{LEGEND_MARKERS} {LEGEND_VERBOSE_HINT}"))); + } + lines.push(String::new()); + lines.extend(diagnostic_lines( + style, + &report.diagnostics, + request.verbose, + )); + lines.push(completeness_line(style, report.complete)); + joined(lines) +} + +/// The child definition folders of one folder, with follow-up commands for each. +pub(crate) fn browse_text( + (style, request): (CliStyle, &ScopeTextRequest), + result: &BrowseResult, +) -> String { + let mut lines: Vec = vec![ + style.section("Scope folders"), + field( + style, + "Path", + &display_path(Some(&result.folder), &request.paths), + ), + String::new(), + ]; + if result.folders.is_empty() { + lines.push(none_line(style)); + } + for (index, folder) in result.folders.iter().enumerate() { + let kinds: Vec = folder + .kind_counts + .iter() + .map(|(kind, count)| format!("{kind} {count}")) + .collect(); + let kinds: String = if kinds.is_empty() { + "no declarations".to_owned() + } else { + kinds.join(", ") + }; + let counts: String = format!( + "{}, {} used, {}; {kinds}", + count_noun(folder.descendant_count, "declaration", "declarations"), + folder.used_count, + count_noun(folder.child_count, "child", "children"), + ); + lines.push(format!( + " {} {}/ {}", + connector(style, index == result.folders.len() - 1), + folder.name, + style.muted(&counts) + )); + for extra in [ + FollowUp { + browse: Some(&folder.path), + ..FollowUp::default() + }, + FollowUp { + list_path: Some(&folder.path), + ..FollowUp::default() + }, + ] { + lines.push(format!(" {}", style.muted(&follow_up(request, extra)))); + } + } + lines.push(String::new()); + lines.extend(diagnostic_lines( + style, + &result.diagnostics, + request.verbose, + )); + lines.push(completeness_line(style, result.complete)); + joined(lines) +} + +/// One page of declarations beneath a folder, with the command that continues it. +pub(crate) fn list_text( + (style, request): (CliStyle, &ScopeTextRequest), + result: &ListResult, +) -> String { + let section: &ScopeSection = &result.section; + let mut lines: Vec = vec![ + style.section("Scope declarations"), + field( + style, + "Path", + &display_path(Some(&result.folder), &request.paths), + ), + field( + style, + "Showing", + &format!("{} of {}", section.returned, section.total), + ), + String::new(), + ]; + let unused: HashSet = HashSet::new(); + lines.extend(declaration_lines( + (style, request), + &result.declarations, + &DeclarationStyle::marking(&unused), + )); + if let Some(cursor) = §ion.next_cursor { + let command: String = follow_up( + request, + FollowUp { + list_path: Some(&result.folder), + after: Some(cursor), + ..FollowUp::default() + }, + ); + lines.push(String::new()); + lines.push(format!("{} {command}", style.muted("Continue:"))); + } + if !request.verbose && !result.declarations.is_empty() { + lines.push(String::new()); + lines.push(style.muted(LEGEND_VERBOSE_HINT)); + } + lines.push(String::new()); + lines.extend(diagnostic_lines( + style, + &result.diagnostics, + request.verbose, + )); + lines.push(completeness_line(style, section.complete)); + joined(lines) +} diff --git a/crates/sqlbuild-scopes/src/scope_text/constants.rs b/crates/sqlbuild-scopes/src/scope_text/constants.rs new file mode 100644 index 0000000000..4c7fee33d7 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/constants.rs @@ -0,0 +1,25 @@ +//! Markers, labels and option defaults of `sqb scope` text output. + +pub(crate) const USED_MARKER: &str = "\u{25cf}"; +pub(crate) const UNUSED_MARKER: &str = "\u{25cb}"; +pub(crate) const LEGEND_MARKERS: &str = + "\u{25cf} used by this resource \u{25cb} not used by this resource"; +pub(crate) const LEGEND_VERBOSE_HINT: &str = "--verbose for scope details"; +pub(crate) const BRANCH_CONNECTOR: &str = "\u{251c}\u{2500}"; +pub(crate) const LAST_CONNECTOR: &str = "\u{2514}\u{2500}"; +pub(crate) const CURRENT_DIRECTORY: &str = "."; +pub(crate) const NONE_TEXT: &str = "(none)"; +pub(crate) const PATH_RELATIVE: &str = "relative"; +pub(crate) const PATH_COMPACT: &str = "compact"; +pub(crate) const PATH_NONE: &str = "none"; +pub(crate) const GLOBAL_SUMMARY: &str = "summary"; +pub(crate) const GLOBAL_ALL: &str = "all"; +pub(crate) const DEFAULT_PAGE_SIZE: i64 = 100; +pub(crate) const ERROR_SEVERITY: &str = "error"; +/// Scope kinds and the label each reads as. +pub(crate) const SCOPE_LABELS: [(&str, &str); 4] = [ + ("global", "project"), + ("inherited", "descendant-public"), + ("local", "exact-owner-private"), + ("private", "model-private"), +]; diff --git a/crates/sqlbuild-scopes/src/scope_text/main/mod.rs b/crates/sqlbuild-scopes/src/scope_text/main/mod.rs new file mode 100644 index 0000000000..e75fa9019f --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/main/mod.rs @@ -0,0 +1,2 @@ +pub mod render_scope_text; +pub mod scope_result_failed; diff --git a/crates/sqlbuild-scopes/src/scope_text/main/render_scope_text.rs b/crates/sqlbuild-scopes/src/scope_text/main/render_scope_text.rs new file mode 100644 index 0000000000..38d8a94912 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/main/render_scope_text.rs @@ -0,0 +1,21 @@ +//! The human text of one `sqb scope` result. + +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::scope_report::models::ScopeResult; +use crate::scope_text::_helpers::results::{browse_text, list_text, report_text}; +use crate::scope_text::models::ScopeTextRequest; + +/// Render a compiler-owned scope result without deriving scope facts. +#[must_use] +pub fn render_scope_text( + result: &ScopeResult, + request: &ScopeTextRequest, + style: CliStyle, +) -> String { + match result { + ScopeResult::Report(report) => report_text((style, request), report), + ScopeResult::Browse(browse) => browse_text((style, request), browse), + ScopeResult::List(list) => list_text((style, request), list), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_text/main/scope_result_failed.rs b/crates/sqlbuild-scopes/src/scope_text/main/scope_result_failed.rs new file mode 100644 index 0000000000..cef9c691cd --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/main/scope_result_failed.rs @@ -0,0 +1,15 @@ +//! Whether a scope result makes `sqb scope` exit with failure. + +use crate::scope_report::models::ScopeResult; +use crate::scope_text::_helpers::layout::has_error; + +/// A result fails when it is partial or carries an error diagnostic. +#[must_use] +pub fn scope_result_failed(result: &ScopeResult) -> bool { + let (complete, diagnostics) = match result { + ScopeResult::Report(report) => (report.complete, &report.diagnostics), + ScopeResult::Browse(browse) => (browse.complete, &browse.diagnostics), + ScopeResult::List(list) => (list.section.complete, &list.diagnostics), + }; + !complete || has_error(diagnostics) +} diff --git a/crates/sqlbuild-scopes/src/scope_text/mod.rs b/crates/sqlbuild-scopes/src/scope_text/mod.rs new file mode 100644 index 0000000000..1644773e62 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-scopes/src/scope_text/models.rs b/crates/sqlbuild-scopes/src/scope_text/models.rs new file mode 100644 index 0000000000..667a5ae044 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/models.rs @@ -0,0 +1,22 @@ +//! The `sqb scope` options its text output reflects. + +/// The command options a scope text rendering shows and repeats in follow-up commands. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ScopeTextRequest { + pub target: Option, + pub at: Option, + pub as_path: Option, + pub defined_under: Option, + pub kinds: Vec, + pub glob: Option, + pub used_only: bool, + pub include_nearby: bool, + pub nearby_depth: i64, + pub dependency_depth: i64, + pub explain: Option, + pub globals: String, + pub page_size: i64, + pub paths: String, + pub verbose: bool, + pub no_cache: bool, +} diff --git a/crates/sqlbuild-scopes/src/scope_text/tests.rs b/crates/sqlbuild-scopes/src/scope_text/tests.rs new file mode 100644 index 0000000000..58333991ff --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_scope_text.rs"] +mod scope_text; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-scopes/src/scope_text/tests/helpers.rs b/crates/sqlbuild-scopes/src/scope_text/tests/helpers.rs new file mode 100644 index 0000000000..5739600401 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/tests/helpers.rs @@ -0,0 +1,53 @@ +use crate::scope_report::models::ScopeResult; +use crate::scope_report::models::{BrowseResult, ListResult, ScopeFolder, ScopeSection}; +use crate::scope_text::models::ScopeTextRequest; + +/// The request of `sqb scope orders` with default options. +pub(super) fn orders_request() -> ScopeTextRequest { + ScopeTextRequest { + target: Some("model:orders".to_owned()), + nearby_depth: 1, + globals: "summary".to_owned(), + page_size: 100, + paths: "relative".to_owned(), + ..ScopeTextRequest::default() + } +} + +/// A browse of `models` with one child folder. +pub(super) fn browse_result(complete: bool) -> ScopeResult { + ScopeResult::Browse(BrowseResult { + folder: "models".to_owned(), + folders: vec![ScopeFolder { + path: "models/my marts".to_owned(), + name: "my marts".to_owned(), + descendant_count: 1, + child_count: 0, + used_count: 1, + kind_counts: vec![("macro".to_owned(), 1)], + }], + diagnostics: Vec::new(), + complete, + }) +} + +/// An empty, complete list page with a continuation cursor. +pub(super) fn list_result() -> ScopeResult { + ScopeResult::List(ListResult { + folder: ".".to_owned(), + declarations: Vec::new(), + section: ScopeSection { + name: "declarations".to_owned(), + total: 2, + returned: 0, + collapsed: false, + collapsed_count: 0, + truncated: true, + complete: true, + next_cursor: Some("c1".to_owned()), + cursor: None, + page_size: 1, + }, + diagnostics: Vec::new(), + }) +} diff --git a/crates/sqlbuild-scopes/src/scope_text/tests/test_scope_text.rs b/crates/sqlbuild-scopes/src/scope_text/tests/test_scope_text.rs new file mode 100644 index 0000000000..75d589f51b --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/tests/test_scope_text.rs @@ -0,0 +1,55 @@ +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::scope_text::main::render_scope_text::render_scope_text; +use crate::scope_text::main::scope_result_failed::scope_result_failed; +use crate::scope_text::models::ScopeTextRequest; +use crate::scope_text::tests::helpers::{browse_result, list_result, orders_request}; +use crate::scope_text::tests::test_types::ScopeTextTestCase; + +#[test] +fn given_scope_result_when_rendering_text_then_text_and_exit_match() { + let test_cases = [ + ScopeTextTestCase { + description: "a browse quotes follow-up paths and reports partial results", + result: browse_result(false), + request: orders_request(), + expected_text: "Scope folders\n Path: models\n\n \u{2514}\u{2500} my marts/ \ + 1 declaration, 1 used, 0 children; macro 1\n \ + sqb scope model:orders --browse 'models/my marts'\n \ + sqb scope model:orders --list 'models/my marts'\n\nCompleteness: partial\n", + expected_failed: true, + }, + ScopeTextTestCase { + description: "a list page offers the command that continues it", + result: list_result(), + request: ScopeTextRequest { + page_size: 1, + verbose: true, + ..orders_request() + }, + expected_text: "Scope declarations\n Path: .\n Showing: 0 of 2\n\n (none)\n\n\ + Continue: sqb scope model:orders --list . --page-size 1 --after c1 --verbose\n\n\ + Diagnostics (0)\n (none)\nCompleteness: complete\n", + expected_failed: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + render_scope_text( + &test_case.result, + &test_case.request, + CliStyle { use_color: false } + ), + scope_result_failed(&test_case.result) + ), + ( + test_case.expected_text.to_owned(), + test_case.expected_failed + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_text/tests/test_types.rs b/crates/sqlbuild-scopes/src/scope_text/tests/test_types.rs new file mode 100644 index 0000000000..4f6210b91f --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/tests/test_types.rs @@ -0,0 +1,10 @@ +use crate::scope_report::models::ScopeResult; +use crate::scope_text::models::ScopeTextRequest; + +pub(super) struct ScopeTextTestCase { + pub(super) description: &'static str, + pub(super) result: ScopeResult, + pub(super) request: ScopeTextRequest, + pub(super) expected_text: &'static str, + pub(super) expected_failed: bool, +} diff --git a/crates/sqlbuild-scopes/src/scope_text/types.rs b/crates/sqlbuild-scopes/src/scope_text/types.rs new file mode 100644 index 0000000000..b721f1aeea --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_text/types.rs @@ -0,0 +1,4 @@ +//! Type aliases of scope text rendering. + +/// A text styling function. +pub type Styler<'a> = &'a dyn Fn(&str) -> String; diff --git a/fensu.toml b/fensu.toml index 53b831b777..046d307047 100644 --- a/fensu.toml +++ b/fensu.toml @@ -28,6 +28,11 @@ rule = "FFR001" path = "src/sqlbuild/cli/commands/models.py" reason = "The TYPE_CHECKING block contains annotation-only imports, avoiding executor and integration imports for read-only commands; runtime declarations remain structured models." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/cursor_algebra/__init__.py" +reason = "The native cursor algebra in sqlbuild-cursor owns cursor behavior; this package only declares the cursor value models, sentinels and grain metadata callers import." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" @@ -81,7 +86,7 @@ reason = "Fingerprint hashing and storage run natively; the package keeps the st [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/refactoring/__init__.py" -reason = "Refactorings are planned natively; the package keeps the request, plan and error models the refactor commands read." +reason = "Refactorings run natively; the package keeps the compiled-project input, manual-location model and error classes the native command reads and raises." [[targets.python.rule_exceptions]] rule = "FFR309" @@ -354,6 +359,7 @@ crate_names = [ "sqlbuild-cli", "sqlbuild-config", "sqlbuild-core", + "sqlbuild-cursor", "sqlbuild-discovery", "sqlbuild-model-config", "sqlbuild-pysource", diff --git a/python-boundary.toml b/python-boundary.toml index a968fdfacc..253fe17c20 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 103112 +remaining_lines = 89820 [[stays]] path = "executor" @@ -219,6 +219,14 @@ path = "tasks.py" reason = "public-api" note = "Public facade." +[[handled_native_errors]] +location = "compiler/planner/models.py::CursorBounds._parse_inferred" +note = "A bound that is not integer text is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." + +[[handled_native_errors]] +location = "compiler/planner/_helpers/warehouse/snapshot.py::_parse_warehouse_cursor" +note = "A warehouse cursor value that is not an integer is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." + [[handled_native_errors]] location = "compiler/planner/_helpers/sql_tests/fixture_formatting.py::format_redundant_fixture_nulls" note = "A test file that no longer parses is left unformatted, as before; no Python implementation runs." diff --git a/scripts/setting_help/constants.py b/scripts/setting_help/constants.py index 8b98fee30e..6041387d9e 100644 --- a/scripts/setting_help/constants.py +++ b/scripts/setting_help/constants.py @@ -17,7 +17,6 @@ ) YAML_FILE_TEXT_MODULES: frozenset[str] = frozenset( { - "src/sqlbuild/cli/commands/_helpers/playground/copy.py", "crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs", } ) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 58a73f8d87..b92c6231d8 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,11 +1,11 @@ """Private SQLBuild native engine bindings.""" -from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence, Set +from collections.abc import Callable, Iterable, Mapping, Sequence, Set from contextlib import AbstractContextManager +from contextvars import Token +from datetime import date, datetime from pathlib import Path, PurePath -from typing import Any, Literal, TypedDict, overload - -from _typeshed import SupportsRichComparison +from typing import Any, Literal, TextIO, TypedDict, overload from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -16,11 +16,25 @@ from sqlbuild.adapter.contract.types import ( TablePromotionMode, TypeDialect, ) -from sqlbuild.cli.commands.types import CompileLineageMode +from sqlbuild.adapter.relations.models import InspectionQueryRecord +from sqlbuild.cli.commands.models import ( + FreshnessCommandRequest, + JanitorCommandRequest, + LineageCommandRequest, + PlaygroundCommandRequest, + RefactorCommandRequest, + ScopeCommandRequest, +) +from sqlbuild.cli.commands.types import CompileLineageMode, ScopeIndexLoader from sqlbuild.cli.compile.models import CompileAnalysis from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt -from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags -from sqlbuild.cli.output.models import WrittenTarget +from sqlbuild.cli.entry.models import ( + AuthoredTargetWarehouse, + CompileCommandRequest, + CompileProfileFlags, + ResolvedWarehouse, +) +from sqlbuild.cli.output.models import SkillMaintenanceResult, SkillUpdateResult, WrittenTarget from sqlbuild.compiler.auditing.models import ( AuditGateIdentity, MeasurementContract, @@ -123,7 +137,6 @@ from sqlbuild.compiler.lineage.models import ( ColumnLineageEdge, DirectSemanticColumnUse, ModelColumnLineage, - QualifiedLineageColumn, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.pipeline.models import ( @@ -142,6 +155,7 @@ from sqlbuild.compiler.planner.models import ( MaximumStartSafetyEvidence, PlanOutput, ) +from sqlbuild.compiler.planner.types import CursorGrain from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, PythonNodeIdentity, @@ -178,6 +192,7 @@ from sqlbuild.compiler.scopes.models import ( VisibilityResolution, ) from sqlbuild.compiler.scopes.types import JsonValue, ScopeQueryTarget +from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord from sqlbuild.compiler.sql_analysis.models import ( SqlBindingResult, SqlLexicalSyntax, @@ -190,20 +205,28 @@ from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestPlanningRequest, ) from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow +from sqlbuild.cost.models import CostRunRecord +from sqlbuild.cursor_algebra.models import AlignedInterval, Duration +from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult -from sqlbuild.executor.clone.models import CloneExecutionResult +from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult +from sqlbuild.executor.janitor.models import JanitorExecutionResult, JanitorPlan from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes.models import ( PythonCheckExecutionResult, PythonNodeExecutionResult, ) from sqlbuild.executor.scenario.models import ScenarioRunResult, ScenarioSnapshotCaptureRunResult -from sqlbuild.executor.testing.models import SqlTestExecutionResult +from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult +from sqlbuild.lint.models import HeaderSpan, LintConfig, LintViolation from sqlbuild.lint.types import RelationKeyRow +from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.presentation.models import DisplayOptions +from sqlbuild.presentation.types import CompletionState from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.rule_engine.models import Finding, Rule, RulesResult from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import ( @@ -221,6 +244,7 @@ from sqlbuild.spec.contracts.models import ( SourceLocation, TargetConfig, ) +from sqlbuild.spec.contracts.types import WarehouseGroup from sqlbuild.sql_values.models import SqlValue BUILD_IDENTITY: str @@ -1099,6 +1123,251 @@ def format_plan( include_completion: bool = False, ) -> str: ... +class IntegrationAssetResult: + kind: str + name: str + status: str + action: str | None + target: str | None + origin_relation: str | None + staging_relation: str | None + failed_phase: str | None + loader: str | None + rows_loaded: int | None + materialized: bool | None + future_cursor_safety: dict[str, object] + maximum_start_safety: dict[str, object] + microbatch: dict[str, object] + def as_dict(self) -> dict[str, Any]: ... + +class IntegrationCheckResult: + kind: str + name: str + check_id: str + passed: bool + status: str + dag_check_id: str | None + severity: str | None + asset_name: str | None + attachment_kind: str | None + attached_column_name: str | None + attached_target_name: str | None + run_scope_phase: str | None + row_count: int | None + reused: bool | None + evaluation_mode: str | None + measured_value: float | None + sample_count: int | None + sample_unit: str | None + minimum_samples: int | None + thresholds: dict[str, object] | None + evidence_count: int | None + evidence_truncated: bool | None + audit_definition_name: str | None + audit_description: str | None + def as_dict(self) -> dict[str, Any]: ... + +class IntegrationResultEnvelope: + schema_version: int + record_kind: str + event_id: str + event_sequence: int + event_type: str + occurred_at: str + invocation_id: str + run_id: str + resource_id: str + resource_attempt_id: str + operation_id: str | None + statement_id: str | None + resource_kind: str + resource_name: str + attempt_number: int + duration_ms: int | float | None + output_kind: str + command: str + error_code: str | None + error_type: str | None + skip_code: str | None + skip_mode: str | None + projection_degraded: bool + @property + def asset(self) -> IntegrationAssetResult | None: ... + @property + def checks(self) -> tuple[IntegrationCheckResult, ...]: ... + @staticmethod + def from_json(value: str) -> IntegrationResultEnvelope: ... + @staticmethod + def from_json_without_optional_enrichment(value: str) -> IntegrationResultEnvelope: ... + def to_json(self) -> str: ... + +class ExecutionEventWriter: + def __init__(self, *, path: Path | None = None) -> None: ... + def write_build_result( + self, *, result: object, plan: PlanOutput | None, command: str = "build" + ) -> None: ... + def write_clone_result(self, *, item: CloneItemResult, resource_type: str) -> None: ... + def close(self) -> None: ... + +def write_cost_run(*, project_dir: Path, record: CostRunRecord) -> Path: ... +def read_cost_run(*, project_dir: Path, run_id: str) -> CostRunRecord | None: ... +def list_cost_runs(*, project_dir: Path) -> tuple[CostRunRecord, ...]: ... +def resolve_cost_run(*, project_dir: Path, selector: str) -> CostRunRecord | None: ... +def format_cost_json(record: CostRunRecord) -> str: ... +def persist_cost_run( + *, project_dir: Path, record: CostRunRecord, output_stream: TextIO, stage: str +) -> None: ... +def write_build_cost_breakdown( + *, record: CostRunRecord, output_stream: TextIO, use_color: bool +) -> None: ... +def format_cost_history_json( + *, records: tuple[CostRunRecord, ...], matching_count: int | None = None +) -> str: ... +def cost_output_payload(*, record: CostRunRecord) -> dict[str, object]: ... +def format_cost_breakdown( + *, + record: CostRunRecord, + use_color: bool, + limit: int | None = None, + terminal_width: int | None = None, + show_run_id: bool = True, +) -> str: ... +def format_cost_history( + *, records: tuple[CostRunRecord, ...], matching_count: int, use_color: bool +) -> str: ... +def format_summary_footer( + *, counts: tuple[tuple[str, int], ...], use_color: bool, elapsed: str | None = None +) -> str: ... +def format_count_noun(*, count: int, singular: str, plural: str | None = None) -> str: ... +def format_completion_line( + *, style: CliStyle, state: CompletionState, label: str, summary: str | None = None +) -> str: ... +def format_status_cell(*, style: CliStyle, status: str, width: int = 6) -> str: ... +def format_aligned_name_value( + *, + plain_name: str, + styled_name: str, + value: str, + name_column_width: int, + prefix: str = " ", +) -> str: ... +def resolve_name_column_width( + *, names: list[str] | tuple[str, ...], min_width: int = 20 +) -> int: ... +def append_overflow_line( + *, lines: list[str], total_count: int, visible_count: int, indent: str, options: DisplayOptions +) -> list[str]: ... +def visible_entries[T](*, entries: Sequence[T], options: DisplayOptions) -> Sequence[T]: ... + +class CountHeaderStyle: + def __call__(self, text: str) -> str: ... + +def count_header_style( + *, style: CliStyle, title_style: Callable[[str], str] | None = None +) -> CountHeaderStyle: ... + +class CliDocument: + def __init__(self, style: CliStyle) -> None: ... + def header(self, *, text: str, suffix: str | None = None) -> None: ... + def blank(self) -> None: ... + def field(self, *, label: str, value: str, value_padding: str = " ") -> None: ... + def fields( + self, *, rows: tuple[tuple[str, str], ...], label_width: int | None = None + ) -> None: ... + def section(self, text: str) -> None: ... + def title_section(self, text: str) -> None: ... + def line(self, text: str) -> None: ... + def items(self, values: tuple[str, ...]) -> None: ... + def command_line(self, *, prefix: str, command: str, style_command: bool = True) -> None: ... + def commands( + self, *, commands: tuple[str, ...], numbered: bool = False, style_command: bool = True + ) -> None: ... + def render(self, *, trailing_newline: bool = True) -> str: ... + +def update_sqlbuild_skills( + *, + project_dir: Path, + global_install: bool = False, + requested_targets: tuple[str, ...] = (), + force: bool = False, + home_dir: Path | None = None, +) -> SkillUpdateResult: ... +def maintain_sqlbuild_skills(*, project_dir: Path) -> SkillMaintenanceResult: ... +def load_packaged_skill_files() -> dict[str, str]: ... +def run_skills_command( + *, + project_dir: Path | None, + global_install: bool = False, + targets: tuple[str, ...] = (), + force: bool = False, +) -> int: ... +def run_playground_command(request: PlaygroundCommandRequest) -> int: ... +def validate_cli_warehouse(*, cli_warehouse: str | None, project_dir: Path) -> None: ... +def find_target_group_warehouse( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None, + group: WarehouseGroup, +) -> AuthoredTargetWarehouse | None: ... +def resolve_command_warehouse( + *, + discovered_inputs: DiscoveredProjectInputs | None, + selected_target: str | None, + group: WarehouseGroup | None, + cli_warehouse: str | None, + connection_config: dict[str, object], + cli_vars: dict[str, object] | None = None, +) -> ResolvedWarehouse: ... +def apply_command_warehouse( + *, + config: dict[str, object], + adapter_name: str, + project_dir: Path | None, + discovered_inputs: DiscoveredProjectInputs | None, + selected_target: str | None, +) -> dict[str, object]: ... +def describe_connection_warehouse( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None, + connection_config: dict[str, object], +) -> ResolvedWarehouse: ... +def resolve_connection_config( + *, + raw_config: dict[str, object], + project_dir: Path, + adapter_name: str, + discovered_inputs: DiscoveredProjectInputs | None = None, + cli_vars: dict[str, object] | None = None, + selected_target: str | None = None, +) -> dict[str, object]: ... +def resolve_project_connection_config( + *, + discovered_inputs: DiscoveredProjectInputs, + project_dir: Path, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, + require_connection: bool = True, +) -> dict[str, object]: ... +def resolve_target_connection_config( + *, + discovered_inputs: DiscoveredProjectInputs, + project_dir: Path, + target_name: str, + cli_vars: dict[str, object] | None = None, +) -> dict[str, object]: ... +def validate_preview_target( + *, discovered_inputs: DiscoveredProjectInputs, as_target: str | None, command_name: str +) -> None: ... +def run_debug_command( + *, + project_dir: Path | None = None, + no_color: bool = False, + no_connection: bool = False, + selected_target: str | None = None, + json_output: bool = False, +) -> int: ... + class TerminalEventClaim: @property def terminal(self) -> LifecycleEvent: ... @@ -1129,6 +1398,72 @@ class TerminalEventIndexScope: def current_terminal_event_index() -> TerminalEventIndex | None: ... def terminal_event_index_scope(index: TerminalEventIndex) -> TerminalEventIndexScope: ... + +class NativeProgressProjector: + def __init__(self, *, stream: TextIO, use_color: bool, debug: bool = False) -> None: ... + def install(self) -> Token[NativeProgressProjector | None]: ... + def restore(self, token: Token[NativeProgressProjector | None]) -> None: ... + def configure_resources(self, *, ordinals: Mapping[str, int], total: int) -> None: ... + def expect_resource_enrichment(self, *, resource_name: str) -> None: ... + def is_operation_active(self, *, operation_name: str) -> bool: ... + def consume(self, event: LifecycleEvent) -> None: ... + def consume_resource_terminal( + self, + *, + resource_name: str, + resource_id: str | None = None, + resource_attempt_id: str | None = None, + attempt_number: int | None = None, + ) -> float | None: ... + def resource_terminal_duration( + self, + *, + resource_name: str, + resource_id: str | None = None, + resource_attempt_id: str | None = None, + attempt_number: int | None = None, + ) -> float | None: ... + def close(self) -> None: ... + +def current_native_progress_projector() -> NativeProgressProjector | None: ... + +class ConnectionProgressReporter: + def __init__( + self, + *, + adapter_name: str, + stream: TextIO, + blank_line_before_start: bool = False, + blank_line_after_complete: bool = False, + use_color: bool = False, + ) -> None: ... + def on_connection_start(self, connection_count: int) -> None: ... + def on_connection_complete(self, *, connection_count: int, elapsed_seconds: float) -> None: ... + def on_connection_error(self, *, connection_count: int, elapsed_seconds: float) -> None: ... + +class PlanningProgressReporter: + def __init__(self, *, stream: TextIO, use_color: bool = False) -> None: ... + def on_progress(self, message: str) -> None: ... + def start(self, message: str) -> None: ... + def update(self, message: str) -> None: ... + def complete(self, message: str) -> None: ... + def complete_styled(self, message: str) -> None: ... + def error(self, message: str) -> None: ... + def finish(self, *, blank_line_after: bool = False) -> None: ... + +class VerboseInspectionWriter: + def __init__(self) -> None: ... + def write(self, record: InspectionQueryRecord) -> None: ... + def write_total(self) -> None: ... + @staticmethod + def format_record(*, record: InspectionQueryRecord) -> str: ... + +def format_execution_header(*, command: str, target: str | None, concurrency: int) -> str: ... +def write_execution_header( + *, stream: TextIO, command: str, target: str | None, concurrency: int, use_color: bool +) -> None: ... +def format_expectation_name(model_name: str) -> str: ... +def format_expectation_detail(step_result: StepResult) -> str: ... def format_build_execution_json( *, result: BuildExecutionResult, @@ -1810,6 +2145,18 @@ def scope_move_preview( lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath ) -> tuple[MovePreview | None, tuple[ScopeDiagnostic, ...]]: ... def serialize_scope_report(report: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: ... +def run_scope_command( + *, + request: ScopeCommandRequest, + load_scope_index: ScopeIndexLoader | None = None, + output_stream: TextIO | None = None, +) -> int: ... +def render_scope_result( + *, + result: ScopeReport | ScopeBrowseResult | ScopeListResult, + request: ScopeCommandRequest, + use_color: bool = False, +) -> str: ... def scope_metadata_projection(index: ScopeIndex) -> dict[str, JsonValue]: ... def effective_config_target_name( project_config: object, local_config: object, selected: str | None, / @@ -2349,42 +2696,6 @@ def build_direct_semantic_uses( model_names: frozenset[str], /, ) -> tuple[DirectSemanticColumnUse, ...]: ... -def render_dependency_tree[Node: Hashable]( - *, - focus: Node, - deps: dict[Node, list[Node]], - seen: set[Node], - format_node: Callable[[Node], str], - sort_key: Callable[[Node], SupportsRichComparison], - branch_style: Callable[[str], str], - already_shown: Callable[[], str], -) -> list[str]: ... -def render_column_trace_tree[Column, Edge]( - *, - target: Column, - deps: dict[str, list[Edge]], - total: int, - limit: int, - column_id: Callable[[Column], str], - related_column: Callable[[Edge], Column], - format_related: Callable[[Edge], str], - branch_style: Callable[[str], str], - already_shown: Callable[[], str], - note_style: Callable[[str], str], -) -> list[str]: ... -def render_column_trace_limit_note( - *, total: int, limit: int, note_style: Callable[[str], str] -) -> list[str]: ... -def serialize_column( - *, - column: QualifiedLineageColumn, - render_resource_type: Callable[[QualifiedLineageColumn], str], -) -> dict[str, object]: ... -def serialize_column_edge( - *, - edge: ColumnLineageEdge, - render_resource_type: Callable[[QualifiedLineageColumn], str], -) -> dict[str, object]: ... class ProjectColumnLineage: """Project-level column lineage graph with lazily materialized public views.""" @@ -2419,6 +2730,29 @@ class ProjectColumnLineage: def relation_lineage_fingerprint( project_dir: Path, prefix: bytes, interrupted_listing_uncacheable: bool, / ) -> str | None: ... +def lineage_cache_fingerprint( + *, project_dir: Path, cli_vars: dict[str, object] | None +) -> str | None: ... +def run_lineage_command(*, request: LineageCommandRequest) -> int: ... +def run_refactor_command(*, request: RefactorCommandRequest) -> int: ... +def run_janitor_command(*, request: JanitorCommandRequest) -> int: ... +def run_freshness_command(*, request: FreshnessCommandRequest) -> int: ... +def observe_source_freshness_json( + *, + adapter: object, + connection: object, + sources: Sequence[SourceEntry], + observed_at: datetime, + select: Sequence[str] = (), + exclude: Sequence[str] = (), + run_id: str | None = None, + previous_records: Mapping[SourceFreshnessIdentity, SourceFreshnessRecord] | None = None, + previous_records_by_source_name: Mapping[str, SourceFreshnessRecord] | None = None, +) -> str: ... +def render_janitor_plan(*, plan: JanitorPlan, use_color: bool = False) -> str: ... +def render_janitor_completion( + *, result: JanitorExecutionResult, use_color: bool = False +) -> str: ... # Native analysis: SQL test planning glue. def plan_compiled_sql_tests( @@ -2570,6 +2904,82 @@ def _oracle_yaml_load(text: str) -> str: ... def _oracle_toml_load(text: str) -> str: ... # Discovery: header values and authored locations. +def cursor_parse(*, raw: object, cursor_type: str) -> CursorScalar: ... +def cursor_try_parse(*, raw: object, cursor_type: str) -> CursorScalar | None: ... +def cursor_render(*, value: CursorScalar) -> str: ... +def cursor_sentinel_to_token(*, sentinel: Bound) -> str: ... +def cursor_sentinel_from_token(*, token: str) -> BoundSentinel | None: ... +def cursor_compare(*, left: CursorScalar, right: CursorScalar) -> int: ... +def cursor_sort_key(*, raw: object, cursor_type: str) -> datetime | int: ... +def cursor_max_bound[T](*, values: tuple[T, ...] | list[T], cursor_type: str) -> T: ... +def cursor_min_bound[T](*, values: tuple[T, ...] | list[T], cursor_type: str) -> T: ... +def cursor_floor_to_grain(*, value: CursorScalar, grain: CursorGrain) -> CursorScalar: ... +def cursor_next_boundary(*, value: CursorScalar, grain: CursorGrain) -> CursorScalar: ... +def cursor_inclusive_to_exclusive( + *, value: CursorScalar, grain: CursorGrain | None +) -> CursorScalar: ... +def cursor_exclusive_to_inclusive( + *, value: CursorScalar, grain: CursorGrain | None +) -> CursorScalar: ... +def cursor_exclusive_end_from_observed_max( + *, value: CursorScalar, grain: CursorGrain | None +) -> CursorScalar: ... +def cursor_observed_partition( + *, value: CursorScalar, grain: CursorGrain | None +) -> AlignedInterval: ... +def cursor_clamp( + *, interval: AlignedInterval, bounds: AlignedInterval +) -> AlignedInterval | None: ... +def cursor_merge(*, intervals: tuple[AlignedInterval, ...]) -> tuple[AlignedInterval, ...]: ... +def cursor_split_into_batches( + *, interval: AlignedInterval, step: int +) -> tuple[AlignedInterval, ...]: ... +def cursor_cap_from_start[T](*, intervals: tuple[T, ...], count: int) -> tuple[T, ...]: ... +def cursor_cap_from_end[T](*, intervals: tuple[T, ...], count: int) -> tuple[T, ...]: ... +def validate_cursor_interval( + *, start: CursorScalar, end: CursorScalar, grain: CursorGrain | None +) -> None: ... +def parse_cursor_duration(value: str) -> tuple[int, int, int, int, int, int] | None: ... +def shift_by_cursor_duration[M: date](*, moment: M, duration: Duration, direction: int) -> M: ... +def get_config_str(*, values: Mapping[str, object], key: str) -> str | None: ... +def get_config_bool(*, values: Mapping[str, object], key: str) -> bool | None: ... +def get_config_int(*, values: Mapping[str, object], key: str) -> int | None: ... +def get_config_string_tuple( + *, values: Mapping[str, object], key: str +) -> tuple[str, ...] | None: ... +def get_config_cursor_bound(*, values: Mapping[str, object], key: str) -> str | None: ... +def custom_rule_import_closure(*, rule: Rule, project_dir: Path) -> tuple[Path, ...]: ... +def custom_rule_implementation_fingerprint( + *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] | None = None +) -> str: ... +def custom_rule_test_evidence( + *, rule: Rule, project_dir: Path +) -> list[tuple[str, int, int, str]]: ... +def format_rules_result(*, result: RulesResult, json_output: bool) -> str: ... +def with_duplicate_literal_hints( + *, findings: tuple[Finding, ...], project_dir: Path +) -> tuple[Finding, ...]: ... +def scan_headers(*, contents: str, first_only: bool = False) -> tuple[HeaderSpan, ...]: ... +def sql_body_ranges( + *, contents: str, headers: tuple[HeaderSpan, ...] +) -> tuple[tuple[int, int], ...]: ... +def measurement_body_ranges(*, contents: str) -> tuple[tuple[int, int], ...]: ... +def lint_native_headers( + *, contents: str, file_path: Path, headers: tuple[HeaderSpan, ...], config: LintConfig +) -> tuple[LintViolation, ...]: ... +def format_native_headers( + *, contents: str, file_path: Path, config: LintConfig +) -> tuple[str, tuple[LintViolation, ...]]: ... +def prepare_native_header_cache(*, files: dict[Path, str]) -> None: ... +def safe_format_files( + *, files: dict[Path, str], config: LintConfig +) -> tuple[dict[Path, str], dict[Path, str]]: ... +def reject_unparseable_header_rewrites( + *, updated: dict[Path, str], config: LintConfig, faults: list[LintViolation] +) -> tuple[dict[Path, str], list[LintViolation]]: ... +def external_identifiers_for_headers( + *, contents: str, headers: tuple[HeaderSpan, ...] +) -> tuple[str, ...]: ... def prepare_model_header_tokens(headers: Sequence[str], /) -> None: ... def parse_header_values( *, diff --git a/src/sqlbuild/cli/commands/_helpers/audit/execution.py b/src/sqlbuild/cli/commands/_helpers/audit/execution.py index cbbe58907a..fc3409740b 100644 --- a/src/sqlbuild/cli/commands/_helpers/audit/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/audit/execution.py @@ -2,20 +2,18 @@ from __future__ import annotations +from sqlbuild._native import ( + ConnectionProgressReporter, + ExecutionEventWriter, + NativeProgressProjector, + current_native_progress_projector, +) from sqlbuild.cli.commands.models import ( AuditCommandRequest, AuditExecutionPreparation, AuditInvocation, ) -from sqlbuild.cli.output.classes.execution_event_writer import ExecutionEventWriter from sqlbuild.cli.progress.classes.audit_progress_reporter import AuditProgressReporter -from sqlbuild.cli.progress.classes.connection_progress_reporter import ( - ConnectionProgressReporter, -) -from sqlbuild.cli.progress.classes.native_progress_projector import ( - NativeProgressProjector, - current_native_progress_projector, -) from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.cost.classes.cost_context import CostContext diff --git a/src/sqlbuild/cli/commands/_helpers/audit/outputs.py b/src/sqlbuild/cli/commands/_helpers/audit/outputs.py index 8b6537880c..3c4ebdbc23 100644 --- a/src/sqlbuild/cli/commands/_helpers/audit/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/audit/outputs.py @@ -4,14 +4,17 @@ import json -from sqlbuild._native import format_audit_execution_json, write_execution_json_output +from sqlbuild._native import ( + format_audit_execution_json, + format_summary_footer, + write_execution_json_output, +) from sqlbuild.cli.commands.models import AuditCommandRequest, AuditInvocation from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.executor.auditing.main.current_audit_result_projection import ( current_audit_result_projection, ) from sqlbuild.executor.auditing.models import AuditExecutionResult, AuditResultProjection -from sqlbuild.presentation.main.summary_footer import format_summary_footer def write_audit_completion_output( diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py index 55b54ce85d..69fa66a823 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py @@ -4,7 +4,11 @@ from typing import Any, TextIO -from sqlbuild._native import build_project_relation_targets +from sqlbuild._native import ( + ConnectionProgressReporter, + build_project_relation_targets, + write_execution_header, +) from sqlbuild.cli.commands._helpers.build_execution.run_context import ( write_build_run_context, ) @@ -27,10 +31,6 @@ BuildRunOutcome, DirectLifecycleCallbacks, ) -from sqlbuild.cli.progress.classes.connection_progress_reporter import ( - ConnectionProgressReporter, -) -from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.main.cursor_window.typed_cursor_overrides import ( typed_cursor_overrides, diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py b/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py index a1f08dd796..93eb264389 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py @@ -6,6 +6,7 @@ from typing import TextIO from sqlbuild._native import ( + cost_output_payload, format_build_execution_json, format_degradable_build_execution_json, format_plan, @@ -28,7 +29,6 @@ from sqlbuild.cli.target_artifacts.main._write_runtime_target import write_runtime_target from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.cost.classes.run_cost_store import RunCostStore from sqlbuild.cost.models import CostRunRecord from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus @@ -138,7 +138,7 @@ def execution_json_payload_with_degradation( python_check_results=python_check_results, command=command, run_id=run_id, - cost=(None if cost_record is None else RunCostStore.output_payload(record=cost_record)), + cost=(None if cost_record is None else cost_output_payload(record=cost_record)), execution_succeeded=execution_succeeded, ) @@ -156,7 +156,7 @@ def write_no_work_build_json( result=BuildExecutionResult(status=BuildStatus.SUCCESS), plan=pipeline_result.plan_output, run_id=(None if cost_record is None else pipeline_result.project.run_id), - cost=(None if cost_record is None else RunCostStore.output_payload(record=cost_record)), + cost=(None if cost_record is None else cost_output_payload(record=cost_record)), ), json_output=request.json_output, json_output_path=request.json_output_path, diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py b/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py index 1f75fa5a4e..de92b47eac 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py @@ -6,6 +6,7 @@ from datetime import datetime from typing import TextIO +from sqlbuild._native import CliDocument from sqlbuild.cli.commands._helpers.build_execution.exceptional_cost import ( finalize_exceptional_direct_cost, ) @@ -20,7 +21,6 @@ from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.models import PartialBuildPhaseTimings -from sqlbuild.presentation.classes.cli_document import CliDocument from sqlbuild.presentation.classes.cli_style import CliStyle diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py b/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py index e9912e3533..4d5f14ae85 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import TextIO -from sqlbuild.cli.commands._helpers.runtime.warehouses import describe_connection_warehouse +from sqlbuild._native import CliDocument, describe_connection_warehouse from sqlbuild.cli.commands.constants import ( C0_CONTROL_CODE_LIMIT, C1_CONTROL_CODE_LIMIT, @@ -14,7 +14,6 @@ ) from sqlbuild.cli.commands.models import BuildRunContext from sqlbuild.cli.entry.models import ResolvedWarehouse -from sqlbuild.presentation.classes.cli_document import CliDocument from sqlbuild.presentation.classes.cli_style import CliStyle diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py b/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py index f65660587e..3a40e4779c 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py @@ -6,10 +6,9 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, resolve_project_connection_config from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import resolve_project_connection_config from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( BuildCommandRequest, diff --git a/src/sqlbuild/cli/commands/_helpers/check/core.py b/src/sqlbuild/cli/commands/_helpers/check/core.py index 181882f88b..6e84a0da06 100644 --- a/src/sqlbuild/cli/commands/_helpers/check/core.py +++ b/src/sqlbuild/cli/commands/_helpers/check/core.py @@ -7,8 +7,10 @@ from typing import Any, TextIO from sqlbuild._native import ( + NativeProgressProjector, TerminalEventIndex, build_python_relation_targets, + current_native_progress_projector, current_terminal_event_index, transitive_closure_many, ) @@ -19,10 +21,6 @@ resolve_relation_location_qualified_name, ) from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.progress.classes.native_progress_projector import ( - NativeProgressProjector, - current_native_progress_projector, -) from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineResult diff --git a/src/sqlbuild/cli/commands/_helpers/check/execution.py b/src/sqlbuild/cli/commands/_helpers/check/execution.py index 16c4ef547d..973e138dbd 100644 --- a/src/sqlbuild/cli/commands/_helpers/check/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/check/execution.py @@ -5,7 +5,12 @@ from collections.abc import Callable from typing import Any -from sqlbuild._native import build_project_relation_targets +from sqlbuild._native import ( + ExecutionEventWriter, + NativeProgressProjector, + build_project_relation_targets, + current_native_progress_projector, +) from sqlbuild.cli.commands._helpers.check.core import ( build_check_relation_targets, load_results_by_loader_name, @@ -18,11 +23,6 @@ CheckExecutionPreparation, CheckInvocation, ) -from sqlbuild.cli.output.classes.execution_event_writer import ExecutionEventWriter -from sqlbuild.cli.progress.classes.native_progress_projector import ( - NativeProgressProjector, - current_native_progress_projector, -) from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import CompilePipelineResult diff --git a/src/sqlbuild/cli/commands/_helpers/check/outputs.py b/src/sqlbuild/cli/commands/_helpers/check/outputs.py index c9e3228712..ffc4034ea7 100644 --- a/src/sqlbuild/cli/commands/_helpers/check/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/check/outputs.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild._native import write_execution_json_output +from sqlbuild._native import format_summary_footer, write_execution_json_output from sqlbuild.cli.commands._helpers.check.core import format_check_json, write_check_results from sqlbuild.cli.commands.models import ( CheckCommandRequest, @@ -13,7 +13,6 @@ write_python_check_runtime_target, ) from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult -from sqlbuild.presentation.main.summary_footer import format_summary_footer def write_check_completion_output( diff --git a/src/sqlbuild/cli/commands/_helpers/clone/connections.py b/src/sqlbuild/cli/commands/_helpers/clone/connections.py index ac84de597a..0f6a691b29 100644 --- a/src/sqlbuild/cli/commands/_helpers/clone/connections.py +++ b/src/sqlbuild/cli/commands/_helpers/clone/connections.py @@ -5,7 +5,8 @@ import time from typing import Any -from sqlbuild.cli.commands._helpers.runtime.connection import ( +from sqlbuild._native import ( + PlanningProgressReporter, resolve_target_connection_config, ) from sqlbuild.cli.commands.models import ( @@ -13,7 +14,6 @@ CloneConnectionContext, CloneInvocation, ) -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.main.phase_line import format_phase_line from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/cli/commands/_helpers/clone/execution.py b/src/sqlbuild/cli/commands/_helpers/clone/execution.py index 0a369f1528..b8316bb9ce 100644 --- a/src/sqlbuild/cli/commands/_helpers/clone/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/clone/execution.py @@ -5,6 +5,7 @@ import time from typing import TextIO +from sqlbuild._native import ExecutionEventWriter from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.clone.fingerprint_progress import ( create_clone_fingerprint_progress_reporter, @@ -20,7 +21,6 @@ CloneInvocation, CloneRunOutcome, ) -from sqlbuild.cli.output.classes.execution_event_writer import ExecutionEventWriter from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ( diff --git a/src/sqlbuild/cli/commands/_helpers/clone/invocation.py b/src/sqlbuild/cli/commands/_helpers/clone/invocation.py index 932fc9b368..c1a07cb339 100644 --- a/src/sqlbuild/cli/commands/_helpers/clone/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/clone/invocation.py @@ -6,13 +6,12 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import PlanningProgressReporter, discover_project_inputs from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.clone.validation import validate_clone_request from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import CloneCommandRequest, CloneInvocation -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/clone/output.py b/src/sqlbuild/cli/commands/_helpers/clone/output.py index aceb081685..3d0b80a50b 100644 --- a/src/sqlbuild/cli/commands/_helpers/clone/output.py +++ b/src/sqlbuild/cli/commands/_helpers/clone/output.py @@ -4,11 +4,10 @@ import re +from sqlbuild._native import format_completion_line, format_summary_footer from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult from sqlbuild.executor.clone.types import CloneAction, CloneStatus from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.completion_line import format_completion_line -from sqlbuild.presentation.main.summary_footer import format_summary_footer from sqlbuild.presentation.types import CompletionState _QUOTED_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile( diff --git a/src/sqlbuild/cli/commands/_helpers/clone/planning.py b/src/sqlbuild/cli/commands/_helpers/clone/planning.py index 23f93e20c1..82b9b6d6c8 100644 --- a/src/sqlbuild/cli/commands/_helpers/clone/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/clone/planning.py @@ -4,6 +4,7 @@ import time +from sqlbuild._native import PlanningProgressReporter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) @@ -14,7 +15,6 @@ CloneExecutionPreparation, CloneInvocation, ) -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.compiler.pipeline.main.clone import run_clone_pipeline from sqlbuild.compiler.pipeline.models import ClonePipelineConnection, ClonePipelineResult from sqlbuild.compiler.planner.models import ( diff --git a/src/sqlbuild/cli/commands/_helpers/contract/progress.py b/src/sqlbuild/cli/commands/_helpers/contract/progress.py index 779075df08..2a2c008629 100644 --- a/src/sqlbuild/cli/commands/_helpers/contract/progress.py +++ b/src/sqlbuild/cli/commands/_helpers/contract/progress.py @@ -6,12 +6,12 @@ from pathlib import Path from typing import Any, TextIO -from sqlbuild._native import compare_contracts -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.models import AdapterConnectionContext, ContractCommandRequest -from sqlbuild.cli.progress.classes.connection_progress_reporter import ( +from sqlbuild._native import ( ConnectionProgressReporter, + compare_contracts, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.cli.commands.models import AdapterConnectionContext, ContractCommandRequest from sqlbuild.compiler.compile.types import CompiledModelView, CompiledSourceView from sqlbuild.compiler.contract_adoption.models import ContractAdoptionResult, ContractEvidence from sqlbuild.compiler.pipeline.models import ProjectGraph diff --git a/src/sqlbuild/cli/commands/_helpers/cost/collection.py b/src/sqlbuild/cli/commands/_helpers/cost/collection.py index 0b8c8f063a..60f6c3ca12 100644 --- a/src/sqlbuild/cli/commands/_helpers/cost/collection.py +++ b/src/sqlbuild/cli/commands/_helpers/cost/collection.py @@ -5,9 +5,8 @@ import logging from typing import TextIO -from sqlbuild.cli.commands._helpers.cost.output import format_cost_breakdown +from sqlbuild._native import persist_cost_run, write_build_cost_breakdown from sqlbuild.cli.commands.models import BuildCostFinalization -from sqlbuild.cost.classes.run_cost_store import RunCostStore from sqlbuild.cost.constants import DEFAULT_RATE_SOURCE from sqlbuild.cost.models import CostRunRecord, RunCostSummary from sqlbuild.cost.types import CostAwareAdapter, CostCapability, CostStatus @@ -76,23 +75,16 @@ def finalize_build_cost(finalization: BuildCostFinalization) -> CostRunRecord | cost=summary, had_executable_work=finalization.had_executable_work, ) - try: - RunCostStore.write(project_dir=finalization.project_dir, record=record) - except BaseException as error: - _LOGGER.exception("automatic run-cost persistence failed") - _write_warning( - output_stream=finalization.output_stream, - message=( - "Cost telemetry failed during artifact persistence " - f"({type(error).__name__}); build result unchanged." - ), - ) - if not finalization.render: - return record - _render_build_cost( + persist_cost_run( + project_dir=finalization.project_dir, record=record, output_stream=finalization.output_stream, - use_color=finalization.use_color, + stage="artifact persistence", + ) + if not finalization.render: + return record + write_build_cost_breakdown( + record=record, output_stream=finalization.output_stream, use_color=finalization.use_color ) return record @@ -104,31 +96,7 @@ def render_build_cost( if record is None: return - _render_build_cost(record=record, output_stream=output_stream, use_color=use_color) - - -def _render_build_cost(*, record: CostRunRecord, output_stream: TextIO, use_color: bool) -> None: - try: - output_stream.write( - "\n" - + format_cost_breakdown( - record=record, - use_color=use_color, - limit=5, - show_run_id=False, - ) - + "\n" - ) - output_stream.flush() - except Exception as error: - _LOGGER.exception("automatic run-cost output failed") - _write_warning( - output_stream=output_stream, - message=( - "Cost telemetry failed during output rendering " - f"({type(error).__name__}); build result unchanged." - ), - ) + write_build_cost_breakdown(record=record, output_stream=output_stream, use_color=use_color) def _write_warning(*, output_stream: TextIO, message: str) -> None: @@ -167,15 +135,10 @@ def _persist_collection_failure( ), had_executable_work=finalization.had_executable_work, ) - try: - RunCostStore.write(project_dir=finalization.project_dir, record=record) - except BaseException as persistence_error: - _LOGGER.exception("automatic run-cost failure artifact persistence failed") - _write_warning( - output_stream=finalization.output_stream, - message=( - "Cost telemetry failed during failure artifact persistence " - f"({type(persistence_error).__name__}); build result unchanged." - ), - ) + persist_cost_run( + project_dir=finalization.project_dir, + record=record, + output_stream=finalization.output_stream, + stage="failure artifact persistence", + ) return record diff --git a/src/sqlbuild/cli/commands/_helpers/cost/command.py b/src/sqlbuild/cli/commands/_helpers/cost/command.py index e875ad4bc3..b8835e7738 100644 --- a/src/sqlbuild/cli/commands/_helpers/cost/command.py +++ b/src/sqlbuild/cli/commands/_helpers/cost/command.py @@ -9,9 +9,13 @@ from pathlib import Path from typing import Any -from sqlbuild.cli.commands._helpers.cost.output import ( +from sqlbuild._native import ( format_cost_breakdown, format_cost_history, + format_cost_history_json, + format_cost_json, + list_cost_runs, + resolve_cost_run, ) from sqlbuild.cli.commands.constants import ( COST_DEFAULT_DETAIL_SORT, @@ -24,7 +28,6 @@ ) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import CostCommandRequest -from sqlbuild.cost.classes.run_cost_store import RunCostStore from sqlbuild.cost.models import CostRunRecord, ResourceCost from sqlbuild.cost.types import CostStatus from sqlbuild.cursor_algebra.constants import MINUTE_TO_DAY_DURATION_UNITS @@ -74,7 +77,7 @@ def _run_history(*, request: CostCommandRequest, project_dir: Path) -> int: until_is_date: bool = request.until is not None and len(request.until) == ISO_DATE_LENGTH matching_records: tuple[CostRunRecord, ...] = tuple( record - for record in RunCostStore.list(project_dir=project_dir) + for record in list_cost_runs(project_dir=project_dir) if (since is None or record.completed_at >= since) and ( until is None @@ -95,9 +98,7 @@ def _run_history(*, request: CostCommandRequest, project_dir: Path) -> int: ) if limit is not None: records = records[:limit] - payload: str = RunCostStore.format_history_json( - records=records, matching_count=len(matching_records) - ) + payload: str = format_cost_history_json(records=records, matching_count=len(matching_records)) _write_json_output(payload=payload, path=request.json_output_path) if request.json_output: sys.stdout.write(payload + "\n") @@ -142,7 +143,7 @@ def _run_detail(*, request: CostCommandRequest, project_dir: Path) -> int: if limit is None else replace(record, cost=replace(record.cost, resources=resources[:limit])) ) - payload: str = RunCostStore.format_json(json_record) + payload: str = format_cost_json(json_record) _write_json_output(payload=payload, path=request.json_output_path) if request.json_output: sys.stdout.write(payload + "\n") @@ -159,9 +160,9 @@ def _run_detail(*, request: CostCommandRequest, project_dir: Path) -> int: def _resolve_run(*, project_dir: Path, selector: str) -> CostRunRecord: - records: tuple[CostRunRecord, ...] = RunCostStore.list(project_dir=project_dir) + records: tuple[CostRunRecord, ...] = list_cost_runs(project_dir=project_dir) if selector == COST_LATEST_SELECTOR: - latest: CostRunRecord | None = RunCostStore.resolve( + latest: CostRunRecord | None = resolve_cost_run( project_dir=project_dir, selector=selector, ) diff --git a/src/sqlbuild/cli/commands/_helpers/cost/output.py b/src/sqlbuild/cli/commands/_helpers/cost/output.py deleted file mode 100644 index 556cf5d9cf..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/cost/output.py +++ /dev/null @@ -1,281 +0,0 @@ -"""Shared cost breakdown and history presentation.""" - -from __future__ import annotations - -import shutil -from datetime import UTC -from decimal import Decimal - -from sqlbuild.cli.commands.constants import ( - BYTE_SCALE, - BYTE_UNIT, - COST_HISTORY_NUMERIC_COLUMNS, - COST_NARROW_TERMINAL_WIDTH, - COST_RESOURCE_NUMERIC_START_COLUMN, - TERABYTE_UNIT, -) -from sqlbuild.cost.constants import DEFAULT_RATE_SOURCE -from sqlbuild.cost.models import CostRunRecord, ResourceCost, RunCostSummary -from sqlbuild.cost.types import CostStatus -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_cost_breakdown( - *, - record: CostRunRecord, - use_color: bool, - limit: int | None = None, - terminal_width: int | None = None, - show_run_id: bool = True, -) -> str: - """Format one run using the same aligned table for build and drill-down output.""" - - style: CliStyle = CliStyle(use_color=use_color) - width: int = terminal_width or shutil.get_terminal_size(fallback=(100, 24)).columns - heading: str = style.title("Cost") - status: str = _styled_cost_status(style=style, status=record.cost.status) - rate_note: str = ( - style.warning(f"default ${record.cost.usd_per_credit:.2f}/credit") - if record.cost.rate_source == DEFAULT_RATE_SOURCE - else style.muted(f"${record.cost.usd_per_credit:.2f}/credit") - ) - total: str = ( - f"{heading} {style.value(_format_usd(record.cost.estimated_usd))} estimated | " - f"{style.value(_format_credits(record.cost.estimated_compute_credits))} | " - f"{rate_note}" - ) - if not record.cost.resources and record.cost.status != CostStatus.COMPLETE: - status_lines: list[str] = [f"{heading} {status}"] - if show_run_id: - status_lines.append(f"{style.label('Run')} {record.run_id}") - if record.cost.message: - status_lines.append(style.muted(record.cost.message)) - status_lines.extend(style.warning(limitation) for limitation in record.cost.limitations) - status_lines.append( - style.muted(f"Run `sqb cost {record.run_id}` to inspect this run later.") - ) - return "\n".join(status_lines) - lines: list[str] = [total] - if record.cost.status != CostStatus.COMPLETE: - lines.append(status) - if show_run_id: - lines.append(f"{style.label('Run')} {record.run_id}") - resources: tuple[ResourceCost, ...] = ( - record.cost.resources if limit is None else record.cost.resources[:limit] - ) - if resources: - lines.extend( - ( - "", - *_format_resource_rows( - resources=resources, - total=record.cost, - style=style, - width=width, - ), - ) - ) - if limit is not None and len(record.cost.resources) > limit: - remaining: int = len(record.cost.resources) - limit - lines.append( - style.muted( - f"Showing {limit} of {len(record.cost.resources)} by estimated cost; " - f"`sqb cost {record.run_id}` for all models ({remaining} more)." - ) - ) - if record.cost.message: - lines.append(style.muted(record.cost.message)) - if record.cost.rate_source == DEFAULT_RATE_SOURCE: - lines.append( - style.warning("Configure cost.usd_per_credit with your Snowflake contract rate.") - ) - lines.extend(style.warning(limitation) for limitation in record.cost.limitations) - lines.append( - style.muted( - "Estimate uses fair-share visible warehouse busy time; it is not Snowflake-billed " - "credits or invoice reconciliation." - ) - ) - return "\n".join(lines) - - -def format_cost_history( - *, records: tuple[CostRunRecord, ...], matching_count: int, use_color: bool -) -> str: - style: CliStyle = CliStyle(use_color=use_color) - lines: list[str] = [style.title("SQLBuild cost history")] - if not records: - return "\n".join( - ( - lines[0], - style.muted("No persisted cost runs."), - style.muted(f"Showing 0 of {matching_count} matching runs."), - ) - ) - headings: tuple[str, ...] = ( - "Completed (UTC)", - "Run", - "Build", - "Cost", - "Attributed credits", - "Est. USD", - ) - rows: list[tuple[str, ...]] = [ - ( - record.completed_at.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S"), - record.run_id, - record.build_status, - record.cost.status.value, - f"{record.cost.estimated_compute_credits:.6f}", - _format_usd(record.cost.estimated_usd), - ) - for record in records - ] - widths: list[int] = _column_widths(headings=headings, rows=rows) - lines.append(" ".join(heading.ljust(widths[index]) for index, heading in enumerate(headings))) - for row in rows: - lines.append( - " ".join( - ( - value.rjust(widths[index]) - if index in COST_HISTORY_NUMERIC_COLUMNS - else value.ljust(widths[index]) - ) - for index, value in enumerate(row) - ) - ) - lines.append( - style.muted( - f"Showing {len(records)} of {matching_count} matching runs. " - "Run `sqb cost ` for a full breakdown." - ) - ) - return "\n".join(lines) - - -def _format_resource_rows( - *, - resources: tuple[ResourceCost, ...], - total: RunCostSummary, - style: CliStyle, - width: int, -) -> tuple[str, ...]: - if width < COST_NARROW_TERMINAL_WIDTH: - headings: tuple[str, ...] = ("Model", "Attributed credits", "Est. cost") - rows: list[tuple[str, ...]] = [ - ( - _truncate(value=resource.resource_name, limit=32), - f"{resource.estimated_compute_credits:.6f}", - _format_usd(resource.estimated_usd), - ) - for resource in resources - ] - rows.append( - ( - "TOTAL", - f"{total.estimated_compute_credits:.6f}", - _format_usd(total.estimated_usd), - ) - ) - return _format_table_rows( - headings=headings, - rows=rows, - numeric_start=COST_RESOURCE_NUMERIC_START_COLUMN - 1, - style=style, - ) - headings: tuple[str, ...] = ( - "Model", - "Warehouse", - "Busy share", - "Scanned", - "Attributed credits", - "Est. cost", - ) - rows: list[tuple[str, ...]] = [ - ( - _truncate(value=resource.resource_name, limit=36), - _truncate(value=resource.warehouse_name, limit=20), - f"{resource.attributed_seconds:.2f}s", - _format_bytes(resource.bytes_scanned), - f"{resource.estimated_compute_credits:.6f}", - _format_usd(resource.estimated_usd), - ) - for resource in resources - ] - rows.append( - ( - "TOTAL", - "", - f"{total.attributed_seconds:.2f}s", - _format_bytes(total.bytes_scanned), - f"{total.estimated_compute_credits:.6f}", - _format_usd(total.estimated_usd), - ) - ) - return _format_table_rows( - headings=headings, - rows=rows, - numeric_start=COST_RESOURCE_NUMERIC_START_COLUMN, - style=style, - ) - - -def _styled_cost_status(*, style: CliStyle, status: CostStatus) -> str: - if status == CostStatus.COMPLETE: - return style.value(status.value) - if status in {CostStatus.PARTIAL, CostStatus.PENDING}: - return style.warning(status.value) - if status == CostStatus.COLLECTION_FAILED: - return style.error(status.value) - return style.muted(status.value) - - -def _format_usd(value: Decimal) -> str: - return f"${value:.4f}" - - -def _format_credits(value: Decimal) -> str: - return f"{value:.6f} attributed credits" - - -def _format_bytes(value: int) -> str: - size: Decimal = Decimal(value) - for unit in ("B", "KiB", "MiB", "GiB", "TiB"): - if size < BYTE_SCALE or unit == TERABYTE_UNIT: - return f"{size:.1f} {unit}" if unit != BYTE_UNIT else f"{int(size)} {BYTE_UNIT}" - size /= Decimal(BYTE_SCALE) - return f"{value} B" - - -def _truncate(*, value: str, limit: int) -> str: - return value if len(value) <= limit else value[: limit - 3] + "..." - - -def _column_widths(*, headings: tuple[str, ...], rows: list[tuple[str, ...]]) -> list[int]: - widths: list[int] = [] - for index, heading in enumerate(headings): - column_widths: list[int] = [len(row[index]) for row in rows] - widths.append(max(len(heading), *column_widths)) - return widths - - -def _format_table_rows( - *, - headings: tuple[str, ...], - rows: list[tuple[str, ...]], - numeric_start: int, - style: CliStyle, -) -> tuple[str, ...]: - widths: list[int] = _column_widths(headings=headings, rows=rows) - output: list[str] = [ - style.label(" ".join(value.ljust(widths[index]) for index, value in enumerate(headings))) - ] - for row in rows: - cells: list[str] = [] - for index, value in enumerate(row): - padded: str = ( - value.rjust(widths[index]) if index >= numeric_start else value.ljust(widths[index]) - ) - cells.append(style.object_name(padded) if index == 0 else style.value(padded)) - output.append(" ".join(cells)) - return tuple(output) diff --git a/src/sqlbuild/cli/commands/_helpers/cost/refresh.py b/src/sqlbuild/cli/commands/_helpers/cost/refresh.py index f3c3619242..edd452b0ac 100644 --- a/src/sqlbuild/cli/commands/_helpers/cost/refresh.py +++ b/src/sqlbuild/cli/commands/_helpers/cost/refresh.py @@ -3,16 +3,18 @@ from dataclasses import replace from pathlib import Path -from sqlbuild._native import discover_project_inputs -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( +from sqlbuild._native import ( + discover_project_inputs, + read_cost_run, + resolve_cost_run, resolve_project_connection_config, + write_cost_run, ) +from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.constants import COST_HISTORY_SELECTOR, COST_LATEST_SELECTOR from sqlbuild.cli.commands.models import CostCommandRequest from sqlbuild.compiler.compile.main.effective_target import build_effective_target_config from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.cost.classes.run_cost_store import RunCostStore from sqlbuild.cost.constants import LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX from sqlbuild.cost.models import CostRunRecord, RunCostSummary from sqlbuild.cost.types import CostAwareAdapter, CostCapability, CostStatus @@ -84,18 +86,18 @@ def refresh_pending_cost_run(request: CostCommandRequest) -> None: rate_source=record.cost.rate_source, ) if _is_non_destructive_refresh(current=record.cost, candidate=summary): - RunCostStore.write(project_dir=project_dir, record=replace(record, cost=summary)) + write_cost_run(project_dir=project_dir, record=replace(record, cost=summary)) except BaseException: return def _resolve_refresh_record(*, project_dir: Path, selector: str) -> CostRunRecord | None: if selector == COST_LATEST_SELECTOR: - return RunCostStore.resolve(project_dir=project_dir, selector=selector) - exact: CostRunRecord | None = RunCostStore.read(project_dir=project_dir, run_id=selector) + return resolve_cost_run(project_dir=project_dir, selector=selector) + exact: CostRunRecord | None = read_cost_run(project_dir=project_dir, run_id=selector) if exact is not None: return exact - return RunCostStore.resolve(project_dir=project_dir, selector=selector) + return resolve_cost_run(project_dir=project_dir, selector=selector) def _is_non_destructive_refresh(*, current: RunCostSummary, candidate: RunCostSummary) -> bool: diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/auto_init.py b/src/sqlbuild/cli/commands/_helpers/dbt/auto_init.py index 25316f82ef..b05f7dd5c9 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/auto_init.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/auto_init.py @@ -6,11 +6,11 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import CliDocument from sqlbuild.cli.commands.classes.dbt_init_progress_reporter import DbtInitProgressReporter from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME from sqlbuild.integrations.dbt.main.profile.profile_init import run_dbt_profile_init from sqlbuild.integrations.dbt.models import DbtInitProgressCallbacks, DbtInitRequest, DbtInitResult -from sqlbuild.presentation.classes.cli_document import CliDocument from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py index 8b8ad248b1..5e360c7595 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import ConnectionProgressReporter, format_execution_header from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.build_planning.plan_policies import ( dbt_interop_plan_safety_gates, @@ -16,8 +17,6 @@ format_build_footer, ) from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter -from sqlbuild.cli.progress.main._execution_header import format_execution_header from sqlbuild.cli.target_artifacts.main._write_runtime_target import write_runtime_target from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.models import ( diff --git a/src/sqlbuild/cli/commands/_helpers/dbt_init/outputs.py b/src/sqlbuild/cli/commands/_helpers/dbt_init/outputs.py index cec2eb8b18..4c4f1b2045 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt_init/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt_init/outputs.py @@ -4,9 +4,9 @@ import sys +from sqlbuild._native import CliDocument from sqlbuild.cli.commands.models import DbtInitCommandRequest from sqlbuild.integrations.dbt.models import DbtInitResult -from sqlbuild.presentation.classes.cli_document import CliDocument from sqlbuild.presentation.classes.cli_style import CliStyle diff --git a/src/sqlbuild/cli/commands/_helpers/debug/__init__.py b/src/sqlbuild/cli/commands/_helpers/debug/__init__.py deleted file mode 100644 index 62c7014540..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/debug/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Debug command helper package.""" diff --git a/src/sqlbuild/cli/commands/_helpers/debug/checks.py b/src/sqlbuild/cli/commands/_helpers/debug/checks.py deleted file mode 100644 index 5bcbb73e0d..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/debug/checks.py +++ /dev/null @@ -1,415 +0,0 @@ -"""Debug command check execution helpers.""" - -from __future__ import annotations - -import platform -import sys -from importlib.metadata import PackageNotFoundError, version -from pathlib import Path - -from sqlbuild._native import discover_project_inputs -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, -) -from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import ( - command_warehouse_scope, - current_command_warehouse_scope, -) -from sqlbuild.cli.commands._helpers.runtime.warehouses import ( - apply_command_warehouse, - find_target_group_warehouse, - resolve_command_warehouse, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import DebugLine, DebugResult -from sqlbuild.cli.commands.types import DebugCheckStatus -from sqlbuild.cli.entry.models import ( - AuthoredTargetWarehouse, - CommandWarehouseScope, - ResolvedWarehouse, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main.effective_target_namespace import ( - build_effective_target_namespace, -) -from sqlbuild.compiler.discovery.constants import ( - LEGACY_LOCAL_CONFIG_FILENAME, - LEGACY_PROJECT_CONFIG_FILENAME, - LOCAL_CONFIG_FILENAME, - PROJECT_CONFIG_FILENAME, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import TargetConfig -from sqlbuild.spec.contracts.types import WarehouseGroup - -_SECRET_CONNECTION_KEYS: frozenset[str] = frozenset( - { - "password", - "private_key", - "private_key_file_pwd", - "secret", - "token", - } -) - - -def build_debug_result( - *, project_dir: Path, check_connection: bool, selected_target: str | None = None -) -> DebugResult: - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - project_config_path: Path = _resolve_existing_path( - project_dir=project_dir, - preferred_filename=PROJECT_CONFIG_FILENAME, - legacy_filename=LEGACY_PROJECT_CONFIG_FILENAME, - ) - local_config_path: Path = _resolve_existing_path( - project_dir=project_dir, - preferred_filename=LOCAL_CONFIG_FILENAME, - legacy_filename=LEGACY_LOCAL_CONFIG_FILENAME, - ) - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter(adapter_name=adapter_name, project_dir=project_dir) - command_scope: CommandWarehouseScope = current_command_warehouse_scope() - with command_warehouse_scope(command=None, cli_warehouse=None): - base_connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - selected_target=selected_target, - ) - connection_config: dict[str, object] - try: - connection_config = apply_command_warehouse( - config=base_connection_config, - adapter_name=adapter_name, - project_dir=project_dir, - discovered_inputs=discovered_inputs, - selected_target=selected_target, - ) - except (CompileInputError, CliUserError): - connection_config = base_connection_config - target_name: str | None - target_config: TargetConfig | None - target_database: str | None - target_schema: str | None - target_name, target_config, target_database, target_schema = build_effective_target_namespace( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - connection_database=connection_config.get("database"), - connection_schema=connection_config.get("schema"), - default_database=adapter.default_database(), - default_schema=adapter.default_schema(), - ) - runtime: list[DebugLine] = _build_runtime_lines() - configuration: list[DebugLine] = [ - DebugLine( - label="project file", - message=str(project_config_path), - status=DebugCheckStatus.OK, - status_message="found and valid", - ), - DebugLine( - label="local config", - message=str(local_config_path) if local_config_path.exists() else "not present", - status=DebugCheckStatus.OK, - status_message="found" if local_config_path.exists() else "not present", - ), - DebugLine( - label="project", - message=discovered_inputs.project_config.name, - status=DebugCheckStatus.OK, - status_message="loaded", - ), - DebugLine( - label="adapter", - message=adapter_name, - status=DebugCheckStatus.OK, - status_message="found", - ), - DebugLine( - label="target", - message=target_name or "default", - status=DebugCheckStatus.OK, - status_message="resolved", - ), - DebugLine( - label="connection", - message=( - target_config.connection_name - if target_config is not None and target_config.connection_name is not None - else "inline" - ), - status=DebugCheckStatus.OK, - status_message="resolved", - ), - DebugLine( - label="database", - message=_display_target_value(target_database), - status=DebugCheckStatus.OK, - status_message="resolved", - ), - DebugLine( - label="schema", - message=_display_target_value(target_schema), - status=DebugCheckStatus.OK, - status_message="resolved", - ), - *_build_warehouse_lines( - adapter=adapter, - discovered_inputs=discovered_inputs, - selected_target=selected_target, - command_scope=command_scope, - base_connection_config=base_connection_config, - ), - ] - connection: list[DebugLine] = _build_connection_config_lines(connection_config) - connection = _append_connection_checks( - connection_lines=connection, - adapter=adapter, - connection_config=connection_config, - check_connection=check_connection, - ) - return DebugResult( - runtime=tuple(runtime), - configuration=tuple(configuration), - providers=tuple(_build_provider_lines(discovered_inputs.providers)), - connection=tuple(connection), - ) - - -def _build_warehouse_lines( - *, - adapter: BaseAdapter, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - command_scope: CommandWarehouseScope, - base_connection_config: dict[str, object], -) -> list[DebugLine]: - lines: list[DebugLine] = [] - group: WarehouseGroup - for group in WarehouseGroup: - authored: AuthoredTargetWarehouse | None = find_target_group_warehouse( - discovered_inputs=discovered_inputs, selected_target=selected_target, group=group - ) - if not adapter.supports_session_warehouse: - if authored is not None: - lines.append( - DebugLine( - label=f"{group.value} warehouse", - message=f"not used (adapter {adapter.adapter_name})", - status=DebugCheckStatus.SKIP, - status_message=authored.source, - ) - ) - continue - lines.append( - _build_warehouse_line( - group=group, - discovered_inputs=discovered_inputs, - selected_target=selected_target, - command_scope=command_scope, - base_connection_config=base_connection_config, - ) - ) - return lines - - -def _build_warehouse_line( - *, - group: WarehouseGroup, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - command_scope: CommandWarehouseScope, - base_connection_config: dict[str, object], -) -> DebugLine: - try: - resolved: ResolvedWarehouse = resolve_command_warehouse( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - group=group, - cli_warehouse=command_scope.cli_warehouse, - connection_config=base_connection_config, - cli_vars=command_scope.cli_vars, - ) - except (CompileInputError, CliUserError) as error: - return DebugLine( - label=f"{group.value} warehouse", - message=f"error: {error.message if isinstance(error, CliUserError) else error}", - status=DebugCheckStatus.ERROR, - status_message="unresolved", - ) - return DebugLine( - label=f"{group.value} warehouse", - message=resolved.warehouse or "not set", - status=DebugCheckStatus.OK, - status_message=resolved.source, - ) - - -def _resolve_existing_path( - *, project_dir: Path, preferred_filename: str, legacy_filename: str -) -> Path: - preferred_path: Path = project_dir / preferred_filename - if preferred_path.exists(): - return preferred_path - legacy_path: Path = project_dir / legacy_filename - if legacy_path.exists(): - return legacy_path - return preferred_path - - -def _build_runtime_lines() -> list[DebugLine]: - try: - sqlbuild_version: str = version("sqlbuild") - except PackageNotFoundError: - sqlbuild_version = "unknown" - return [ - DebugLine(label="sqlbuild version", message=sqlbuild_version), - DebugLine(label="python version", message=platform.python_version()), - DebugLine(label="python path", message=sys.executable), - DebugLine(label="os info", message=platform.platform()), - ] - - -def _display_target_value(value: object | None) -> str: - return value if isinstance(value, str) and value else "not configured" - - -def _build_connection_config_lines(connection_config: dict[str, object]) -> list[DebugLine]: - if not connection_config: - return [DebugLine(label="settings", message="no keys", status=DebugCheckStatus.OK)] - return [ - DebugLine(label=key, message=_sanitize_connection_value(key=key, value=value)) - for key, value in sorted(connection_config.items()) - ] - - -def _build_provider_lines(providers: tuple[DiscoveredProvider, ...]) -> list[DebugLine]: - if not providers: - return [ - DebugLine( - label="providers", - message="none discovered", - status=DebugCheckStatus.OK, - ) - ] - lines: list[DebugLine] = [ - DebugLine( - label="providers", - message=str(len(providers)), - status=DebugCheckStatus.OK, - status_message="discovered", - ) - ] - provider: DiscoveredProvider - for provider in providers: - lines.append( - DebugLine( - label=provider.name, - message=f"{provider.relative_path}:{provider.provider_class.__name__}", - status=DebugCheckStatus.OK, - status_message="valid settings", - ) - ) - return lines - - -def _append_connection_checks( - *, - connection_lines: list[DebugLine], - adapter: BaseAdapter, - connection_config: dict[str, object], - check_connection: bool, -) -> list[DebugLine]: - if not check_connection: - connection_lines.append( - DebugLine( - label="connection test", - message="", - status=DebugCheckStatus.SKIP, - status_message="skipped by --no-connection", - ) - ) - connection_lines.append( - DebugLine( - label="query test", - message="", - status=DebugCheckStatus.SKIP, - status_message="connection skipped", - ) - ) - return connection_lines - - connection: object | None = None - try: - connection = adapter.connect(connection_config) - connection_lines.append( - DebugLine( - label="connection test", - message="", - status=DebugCheckStatus.OK, - status_message="connected", - ) - ) - except Exception as error: - connection_lines.append( - DebugLine( - label="connection test", - message="", - status=DebugCheckStatus.ERROR, - status_message=str(error), - ) - ) - connection_lines.append( - DebugLine( - label="query test", - message="", - status=DebugCheckStatus.SKIP, - status_message="connection failed", - ) - ) - return connection_lines - - try: - adapter.query(connection=connection, sql="SELECT 1", limit=None) - connection_lines.append( - DebugLine( - label="query test", - message="", - status=DebugCheckStatus.OK, - status_message="SELECT 1", - ) - ) - except Exception as error: - connection_lines.append( - DebugLine( - label="query test", - message="", - status=DebugCheckStatus.ERROR, - status_message=str(error), - ) - ) - finally: - adapter.close(connection) - return connection_lines - - -def _sanitize_connection_value(*, key: str, value: object) -> str: - if key.lower() in _SECRET_CONNECTION_KEYS: - return "****" - if value is None: - return "null" - if isinstance(value, bool | int | float): - return str(value) - text: str = str(value) - digest_preview_character_limit: int = 32 - if len(text) <= digest_preview_character_limit: - return text - return f"{text[:4]}...{text[-4:]}" diff --git a/src/sqlbuild/cli/commands/_helpers/debug/output.py b/src/sqlbuild/cli/commands/_helpers/debug/output.py deleted file mode 100644 index 1103cea567..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/debug/output.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Debug command output helpers.""" - -from __future__ import annotations - -import json - -from sqlbuild.cli.commands.models import DebugLine, DebugResult -from sqlbuild.cli.commands.types import DebugCheckStatus -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_debug_text(*, result: DebugResult, use_color: bool) -> str: - style: CliStyle = CliStyle(use_color=use_color) - header: str = style.title("SQLBuild Diagnostics") - lines: list[str] = ["", header, ""] - lines = _append_section( - lines=lines, section_name="Runtime", section_lines=result.runtime, style=style - ) - lines = _append_section( - lines=lines, section_name="Configuration", section_lines=result.configuration, style=style - ) - lines = _append_section( - lines=lines, section_name="Providers", section_lines=result.providers, style=style - ) - lines = _append_section( - lines=lines, section_name="Connection", section_lines=result.connection, style=style - ) - return "\n".join(lines) + "\n" - - -def format_debug_json(result: DebugResult) -> str: - return json.dumps( - { - "success": result.success, - "runtime": [_line_to_json(line) for line in result.runtime], - "configuration": [_line_to_json(line) for line in result.configuration], - "providers": [_line_to_json(line) for line in result.providers], - "connection": [_line_to_json(line) for line in result.connection], - }, - sort_keys=True, - ) - - -def _append_section( - *, lines: list[str], section_name: str, section_lines: tuple[DebugLine, ...], style: CliStyle -) -> list[str]: - rendered_section_name: str = style.section(f"{section_name}:") - lines.append(rendered_section_name) - line: DebugLine - for line in section_lines: - message: str = line.message - if line.status is not None: - status_message: str = line.status_message or line.message - status: str = _format_status(status=line.status, message=status_message, style=style) - message = status if not message else f"{message} {status}" - lines.append(f" {line.label}: {message}") - lines.append("") - return lines - - -def _format_status(*, status: DebugCheckStatus, message: str, style: CliStyle) -> str: - status_text: str = "OK" if status == DebugCheckStatus.OK else status.value - rendered: str = f"[{status_text} {message}]" - return style.status(status=status_text, text=rendered) - - -def _line_to_json(line: DebugLine) -> dict[str, str]: - payload: dict[str, str] = {"label": line.label, "message": line.message} - if line.status is not None: - payload["status"] = line.status.value - if line.status_message is not None: - payload["status_message"] = line.status_message - return payload diff --git a/src/sqlbuild/cli/commands/_helpers/diff/execution.py b/src/sqlbuild/cli/commands/_helpers/diff/execution.py index 577c73c5c3..81400a38dc 100644 --- a/src/sqlbuild/cli/commands/_helpers/diff/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/diff/execution.py @@ -8,7 +8,11 @@ from pathlib import Path from typing import Any -from sqlbuild._native import run_diff_pipeline +from sqlbuild._native import ( + resolve_project_connection_config, + resolve_target_connection_config, + run_diff_pipeline, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.contract.types import BuiltinAdapter @@ -21,10 +25,6 @@ resolve_external_sql_reference_resolver, ) from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, - resolve_target_connection_config, -) from sqlbuild.cli.commands.constants import READ_ONLY_QUERY_ROOT_KEYS from sqlbuild.cli.commands.exceptions import CliUserError, QueryDiffExecutionError from sqlbuild.cli.commands.models import ( diff --git a/src/sqlbuild/cli/commands/_helpers/entry/observability.py b/src/sqlbuild/cli/commands/_helpers/entry/observability.py index 560e7d0b9f..0904aac803 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/observability.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/observability.py @@ -11,6 +11,7 @@ from pathlib import Path from sqlbuild._native import ( + NativeProgressProjector, TerminalEventIndex, terminal_event_index_scope, ) @@ -26,7 +27,6 @@ configured_output_capture_scope, ) from sqlbuild.cli.entry.main._withhold_launch_record import withhold_launch_record -from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector from sqlbuild.compiler.discovery.main.runtime_extensions import discover_runtime_extensions from sqlbuild.compiler.discovery.models import ( DiscoveredCommandOutputSink, diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/observe.py b/src/sqlbuild/cli/commands/_helpers/freshness/observe.py deleted file mode 100644 index 24daf99bae..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/freshness/observe.py +++ /dev/null @@ -1,374 +0,0 @@ -"""Source freshness command observation helpers.""" - -from __future__ import annotations - -from contextlib import AbstractContextManager, nullcontext -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.cli.commands.models import ( - FreshnessCommandResult, - FreshnessSourceResult, -) -from sqlbuild.cli.commands.types import FreshnessSourceStatus -from sqlbuild.compiler.source_freshness.main.adapter_observation import ( - observe_adapter_sources_freshness, -) -from sqlbuild.compiler.source_freshness.main.age_policy import ( - evaluate_source_freshness_age_policy, -) -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) -from sqlbuild.compiler.source_freshness.main.observation import ( - observe_configured_source_freshness, -) -from sqlbuild.compiler.source_freshness.main.record_equivalence import ( - source_freshness_records_equivalent, -) -from sqlbuild.compiler.source_freshness.models import ( - AdapterSourceFreshnessBatch, - SourceFreshnessIdentity, - SourceFreshnessObservation, - SourceFreshnessRecord, - SourceFreshnessUnknown, -) -from sqlbuild.compiler.source_freshness.types import ( - SourceFreshnessAgeStatus, - SourceFreshnessUnknownReason, -) -from sqlbuild.runtime.observability.main.run_scope import run_scope -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy - - -def observe_source_freshness_for_command( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - select: tuple[str, ...] = (), - exclude: tuple[str, ...] = (), - observed_at: datetime, - run_id: str | None = None, - previous_records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] | None = None, - previous_records_by_source_name: dict[str, SourceFreshnessRecord] | None = None, -) -> FreshnessCommandResult: - """Observe current source freshness for the CLI command.""" - - scope: AbstractContextManager[object] = ( - run_scope(run_id) if run_id is not None else nullcontext() - ) - with scope: - return _observe_source_freshness_for_command( - adapter=adapter, - connection=connection, - sources=sources, - select=select, - exclude=exclude, - observed_at=observed_at, - previous_records=previous_records, - previous_records_by_source_name=previous_records_by_source_name, - ) - - -def _observe_source_freshness_for_command( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - select: tuple[str, ...], - exclude: tuple[str, ...], - observed_at: datetime, - previous_records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] | None, - previous_records_by_source_name: dict[str, SourceFreshnessRecord] | None, -) -> FreshnessCommandResult: - selected_sources: tuple[SourceEntry, ...] = _selected_sources( - sources=sources, - select=select, - exclude=exclude, - ) - results: list[FreshnessSourceResult] = [] - previous_by_identity: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = ( - previous_records if previous_records is not None else {} - ) - previous_by_source_name: dict[str, SourceFreshnessRecord] = ( - previous_records_by_source_name if previous_records_by_source_name is not None else {} - ) - compare_state: bool = ( - previous_records is not None or previous_records_by_source_name is not None - ) - observation_sources_by_name: dict[str, SourceEntry] = {} - adapter_observation_sources: list[SourceEntry] = [] - source: SourceEntry - for source in selected_sources: - observation_source: SourceEntry | None = _source_for_observation( - adapter=adapter, - source=source, - ) - if observation_source is None: - results.append( - FreshnessSourceResult( - name=source.name, - status=FreshnessSourceStatus.UNKNOWN, - target_database=source.database, - target_schema=source.schema, - target_name=source.table, - message="no freshness config and adapter metadata unavailable", - ) - ) - continue - observation_sources_by_name[source.name] = observation_source - if observation_source.freshness is not None and ( - observation_source.freshness.strategy == SourceFreshnessStrategy.ADAPTER - ): - adapter_observation_sources.append(observation_source) - - adapter_batch: AdapterSourceFreshnessBatch = AdapterSourceFreshnessBatch() - adapter_observation_error: Exception | None = None - if adapter_observation_sources: - try: - adapter_batch = observe_adapter_sources_freshness( - adapter=adapter, - connection=connection, - sources=tuple(adapter_observation_sources), - observed_at=observed_at, - ) - except Exception as exc: - adapter_observation_error = exc - - for source_name, observation_source in observation_sources_by_name.items(): - if observation_source.freshness is not None and ( - observation_source.freshness.strategy == SourceFreshnessStrategy.ADAPTER - ): - if adapter_observation_error is not None: - results.append( - FreshnessSourceResult( - name=source_name, - status=FreshnessSourceStatus.ERROR, - target_database=observation_source.database, - target_schema=observation_source.schema, - target_name=observation_source.table, - message=str(adapter_observation_error), - ) - ) - continue - source_unknown: SourceFreshnessUnknown | None = adapter_batch.unknown.get(source_name) - if source_unknown is not None: - results.append( - FreshnessSourceResult( - name=source_name, - status=( - FreshnessSourceStatus.UNKNOWN - if source_unknown.reason == SourceFreshnessUnknownReason.UNAVAILABLE - else FreshnessSourceStatus.ERROR - ), - target_database=observation_source.database, - target_schema=observation_source.schema, - target_name=observation_source.table, - message=source_unknown.message, - ) - ) - continue - observation: SourceFreshnessObservation | None = adapter_batch.observations.get( - source_name - ) - if observation is None: - results.append( - FreshnessSourceResult( - name=source_name, - status=FreshnessSourceStatus.ERROR, - target_database=observation_source.database, - target_schema=observation_source.schema, - target_name=observation_source.table, - message="freshness metadata was not returned", - ) - ) - continue - else: - try: - observation = observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=observation_source, - observed_at=observed_at, - ) - except Exception as exc: - results.append( - FreshnessSourceResult( - name=source_name, - status=FreshnessSourceStatus.ERROR, - target_database=observation_source.database, - target_schema=observation_source.schema, - target_name=observation_source.table, - message=str(exc), - ) - ) - continue - record: SourceFreshnessRecord = _record_from_observation( - observation=observation, - source=observation_source, - run_id="freshness", - ) - results.append( - _source_result_from_record( - name=source_name, - current_record=record, - previous_record=previous_by_identity.get(record.identity) - or previous_by_source_name.get(record.source_name), - lag_tolerance=observation_source.freshness.lag_tolerance - if observation_source.freshness is not None - else None, - age_status=evaluate_source_freshness_age_policy( - policy=observation_source.freshness.age_policy - if observation_source.freshness is not None - else None, - data_version=observation.data_version, - observed_at=observation.observed_at, - ), - compare_state=compare_state, - ) - ) - return FreshnessCommandResult(sources=tuple(sorted(results, key=lambda item: item.name))) - - -def _record_from_observation( - *, observation: SourceFreshnessObservation, source: SourceEntry, run_id: str -) -> SourceFreshnessRecord: - normalized_data_version: str = normalize_source_freshness_data_version( - value=observation.data_version, - value_kind=observation.value_kind, - ) - return SourceFreshnessRecord( - source_name=observation.source_name, - target_database=source.database, - target_schema=source.schema, - target_name=source.table, - run_id=run_id, - strategy=observation.strategy.value, - value_kind=observation.value_kind.value, - data_version=normalized_data_version, - data_version_hash=source_freshness_data_version_hash( - source_name=observation.source_name, - strategy=observation.strategy, - value_kind=observation.value_kind, - data_version=normalized_data_version, - ), - observed_at=observation.observed_at, - ) - - -def _source_result_from_record( - *, - name: str, - current_record: SourceFreshnessRecord, - previous_record: SourceFreshnessRecord | None, - lag_tolerance: str | None, - age_status: SourceFreshnessAgeStatus | None, - compare_state: bool, -) -> FreshnessSourceResult: - if previous_record is None: - if compare_state: - return FreshnessSourceResult( - name=name, - status=FreshnessSourceStatus.UNKNOWN, - strategy=current_record.strategy, - value_kind=current_record.value_kind, - current_data_version=current_record.data_version, - lag_tolerance=lag_tolerance, - target_database=current_record.target_database, - target_schema=current_record.target_schema, - target_name=current_record.target_name, - message="previous source freshness state missing", - age_status=age_status, - ) - return FreshnessSourceResult( - name=name, - status=FreshnessSourceStatus.OBSERVED, - strategy=current_record.strategy, - value_kind=current_record.value_kind, - current_data_version=current_record.data_version, - lag_tolerance=lag_tolerance, - target_database=current_record.target_database, - target_schema=current_record.target_schema, - target_name=current_record.target_name, - age_status=age_status, - ) - equivalent: bool = source_freshness_records_equivalent( - previous_record=previous_record, - current_record=current_record, - lag_tolerance=lag_tolerance, - ) - status: FreshnessSourceStatus - if equivalent and previous_record.data_version_hash == current_record.data_version_hash: - status = FreshnessSourceStatus.UNCHANGED - elif equivalent: - status = FreshnessSourceStatus.TOLERATED - else: - status = FreshnessSourceStatus.CHANGED - return FreshnessSourceResult( - name=name, - status=status, - strategy=current_record.strategy, - value_kind=current_record.value_kind, - current_data_version=current_record.data_version, - previous_data_version=previous_record.data_version, - lag_tolerance=lag_tolerance, - target_database=current_record.target_database, - target_schema=current_record.target_schema, - target_name=current_record.target_name, - age_status=age_status, - ) - - -def _selected_sources( - *, sources: tuple[SourceEntry, ...], select: tuple[str, ...], exclude: tuple[str, ...] -) -> tuple[SourceEntry, ...]: - selected_names: frozenset[str] = frozenset(select) - excluded_names: frozenset[str] = frozenset(exclude) - return tuple( - source - for source in sources - if (not selected_names or source.name in selected_names) - and source.name not in excluded_names - ) - - -def _source_for_observation(*, adapter: StrictAdapter, source: SourceEntry) -> SourceEntry | None: - if source.freshness is not None: - return source - if ( - source.expression is None - and source.table is not None - and adapter.supports_table_freshness_metadata() - ): - return _source_with_adapter_freshness(source) - return None - - -def _source_with_adapter_freshness(source: SourceEntry) -> SourceEntry: - return SourceEntry( - name=source.name, - database=source.database, - schema=source.schema, - table=source.table, - loader=source.loader, - managed=source.managed, - integration_loader=source.integration_loader, - freshness=SourceFreshnessConfig(strategy=SourceFreshnessStrategy.ADAPTER), - write_strategy=source.write_strategy, - load_batch_size=source.load_batch_size, - cursor_column=source.cursor_column, - unique_key=source.unique_key, - expression=source.expression, - description=source.description, - type_enforcement=source.type_enforcement, - contract=source.contract, - meta=source.meta, - columns=source.columns, - audits=source.audits, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/output.py b/src/sqlbuild/cli/commands/_helpers/freshness/output.py deleted file mode 100644 index 79fe1d52dd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/freshness/output.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Source freshness command output formatting.""" - -from __future__ import annotations - -import json - -from sqlbuild.cli.commands.models import ( - FreshnessCommandResult, - FreshnessSourceResult, -) -from sqlbuild.cli.commands.types import FreshnessSourceStatus -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.summary_footer import format_summary_footer -from sqlbuild.presentation.main.tree_connector import tree_connector - -_FRESHNESS_GROUPS: tuple[tuple[FreshnessSourceStatus, str], ...] = ( - (FreshnessSourceStatus.OBSERVED, "Observed"), - (FreshnessSourceStatus.CHANGED, "Changed"), - (FreshnessSourceStatus.UNCHANGED, "Unchanged"), - (FreshnessSourceStatus.TOLERATED, "Tolerated"), - (FreshnessSourceStatus.UNKNOWN, "Unknown"), - (FreshnessSourceStatus.ERROR, "Errors"), -) - - -def format_freshness_text(*, result: FreshnessCommandResult, use_color: bool = False) -> str: - """Format source freshness observations for humans.""" - - style: CliStyle = CliStyle(use_color=use_color) - lines: list[str] = [style.title("Source freshness")] - name_width: int = max((len(source.name) for source in result.sources), default=0) - status: FreshnessSourceStatus - title: str - for status, title in _FRESHNESS_GROUPS: - sources: tuple[FreshnessSourceResult, ...] = tuple( - source for source in result.sources if source.status == status - ) - lines = _append_group( - lines=lines, - title=title, - sources=sources, - name_width=name_width, - style=style, - ) - lines.append("") - lines.append( - format_summary_footer( - counts=( - ("OBSERVED", result.observed_count), - ("CHANGED", result.changed_count), - ("UNCHANGED", result.unchanged_count), - ("TOLERATED", result.tolerated_count), - ("UNKNOWN", result.unknown_count), - ("ERROR", result.error_count), - ), - use_color=use_color, - ) - ) - if any(source.age_status is not None for source in result.sources): - lines.append( - f"{style.section('Age policy')} " - + format_summary_footer( - counts=( - ("PASS", result.age_pass_count), - ("WARN", result.age_warn_count), - ("ERROR", result.age_error_count), - ("UNKNOWN", result.age_unknown_count), - ), - use_color=use_color, - ) - ) - return "\n".join(lines) + "\n" - - -def format_freshness_json(result: FreshnessCommandResult) -> str: - """Serialize source freshness observations as JSON.""" - - return json.dumps( - { - "sources": [_source_payload(source) for source in result.sources], - "summary": { - "observed": result.observed_count, - "changed": result.changed_count, - "unchanged": result.unchanged_count, - "tolerated": result.tolerated_count, - "unknown": result.unknown_count, - "errors": result.error_count, - "age_pass": result.age_pass_count, - "age_warn": result.age_warn_count, - "age_error": result.age_error_count, - "age_unknown": result.age_unknown_count, - }, - }, - indent=2, - ) - - -def _append_group( - *, - lines: list[str], - title: str, - sources: tuple[FreshnessSourceResult, ...], - name_width: int, - style: CliStyle, -) -> list[str]: - if not sources: - return lines - lines.append("") - lines.append(style.section(f"{title} ({len(sources)})")) - source: FreshnessSourceResult - for index, source in enumerate(sources): - connector: str = tree_connector(style=style, last=index == len(sources) - 1) - detail: str = _format_source_detail(source=source, style=style) - suffix: str = f" {detail}" if detail else "" - lines.append(f"{connector} {style.object_name(f'{source.name:<{name_width}}')}{suffix}") - return lines - - -def _format_source_detail(*, source: FreshnessSourceResult, style: CliStyle) -> str: - if source.status == FreshnessSourceStatus.OBSERVED: - parts: list[str] = [] - if source.current_data_version is not None: - parts.append(style.muted(f"value {source.current_data_version}")) - if source.value_kind is not None: - parts.append(style.muted(f"kind {source.value_kind}")) - if source.strategy is not None: - parts.append(style.muted(f"via {source.strategy}")) - if source.lag_tolerance is not None: - parts.append(style.muted(f"tolerance {source.lag_tolerance}")) - if source.age_status is not None: - parts.append( - style.status( - status=source.age_status.value, - text=f"age {source.age_status.value}", - ) - ) - return " ".join(parts) - if source.status in { - FreshnessSourceStatus.CHANGED, - FreshnessSourceStatus.UNCHANGED, - FreshnessSourceStatus.TOLERATED, - }: - parts = [] - if source.previous_data_version is not None: - parts.append(style.muted(f"previous {source.previous_data_version}")) - if source.current_data_version is not None: - parts.append(style.muted(f"current {source.current_data_version}")) - if source.lag_tolerance is not None: - parts.append(style.muted(f"tolerance {source.lag_tolerance}")) - if source.age_status is not None: - parts.append( - style.status( - status=source.age_status.value, - text=f"age {source.age_status.value}", - ) - ) - return " ".join(parts) - if source.message is not None: - if source.status == FreshnessSourceStatus.ERROR: - return style.error(source.message) - return style.warning(source.message) - return "" - - -def _source_payload(source: FreshnessSourceResult) -> dict[str, object]: - return { - "name": source.name, - "status": source.status.value, - "strategy": source.strategy, - "value_kind": source.value_kind, - "current_data_version": source.current_data_version, - "previous_data_version": source.previous_data_version, - "lag_tolerance": source.lag_tolerance, - "target": { - "database": source.target_database, - "schema": source.target_schema, - "name": source.target_name, - }, - "message": source.message, - "age_status": source.age_status.value if source.age_status is not None else None, - } diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/selection.py b/src/sqlbuild/cli/commands/_helpers/freshness/selection.py deleted file mode 100644 index dd81d5f74a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/freshness/selection.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Source freshness command selector helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection.upstream import expand_project_upstream_keys - - -def resolve_freshness_source_names( - *, graph: ProjectGraph, select: tuple[str, ...], exclude: tuple[str, ...] -) -> tuple[str, ...]: - """Resolve CLI selectors to source names that should be observed.""" - - selected_keys: frozenset[CompiledObjectKey] - if select: - selected_keys = select_project_graph(graph=graph, select=select, exclude=()) - else: - selected_keys = frozenset(graph.all_keys.values()) - - excluded_keys: frozenset[CompiledObjectKey] = ( - select_project_graph(graph=graph, select=exclude, exclude=()) if exclude else frozenset() - ) - source_names: frozenset[str] = _source_names_for_keys(graph=graph, keys=selected_keys) - excluded_source_names: frozenset[str] = _source_names_for_keys( - graph=graph, - keys=excluded_keys, - ) - return tuple(sorted(source_names - excluded_source_names)) - - -def _source_names_for_keys( - *, graph: ProjectGraph, keys: frozenset[CompiledObjectKey] -) -> frozenset[str]: - source_names: set[str] = set() - key: CompiledObjectKey - for key in keys: - if key.resource_type == CompiledResourceType.SOURCE: - source_names.add(key.name) - upstream_key: CompiledObjectKey - for upstream_key in expand_project_upstream_keys( - key=key, - upstream_deps=graph.upstream_deps, - ): - if upstream_key.resource_type == CompiledResourceType.SOURCE: - source_names.add(upstream_key.name) - return frozenset(source_names) diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/state.py b/src/sqlbuild/cli/commands/_helpers/freshness/state.py deleted file mode 100644 index c072959494..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/freshness/state.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Source freshness command state helpers.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.main.read import read_latest_source_freshness -from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord - - -def read_direct_freshness_state_for_command( - *, adapter: StrictAdapter, connection: Any, project: Any -) -> dict[SourceFreshnessIdentity, SourceFreshnessRecord]: - """Read direct source freshness state for all compiled target schemas.""" - - records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = {} - state_database: str | None = _resolve_state_database(project=project) - state_schemas: tuple[str, ...] = tuple(_collect_state_schemas(project=project)) - state_table_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple( - (state_database, state_schema, SOURCE_FRESHNESS_TABLE_NAME) - for state_schema in state_schemas - ), - ) - state_schema: str - for state_schema in state_schemas: - records.update( - read_latest_source_freshness( - connection=connection, - execute=adapter.execute, - table_exists=state_table_lookup.exists( - database=state_database, - schema=state_schema, - name=SOURCE_FRESHNESS_TABLE_NAME, - ), - database=state_database, - schema=state_schema, - render_qualified_name=adapter.render_qualified_name, - render_read_latest_sql=adapter.render_read_latest_source_freshness_sql, - ).records - ) - return records - - -def _resolve_state_database(*, project: Any) -> str | None: - destination: Any - for destination in _iter_state_destinations(project=project): - if destination.database is not None: - return destination.database - return None - - -def _collect_state_schemas(*, project: Any) -> tuple[str, ...]: - schemas: set[str] = set() - destination: Any - for destination in _iter_state_destinations(project=project): - if destination.schema is not None: - schemas.add(destination.schema) - return tuple(sorted(schemas)) - - -def _iter_state_destinations(*, project: Any) -> tuple[Any, ...]: - return ( - *(model.destination for model in project.models), - *(seed.destination for seed in project.seeds), - *(function.destination for function in project.functions), - ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py b/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py deleted file mode 100644 index d7126bb924..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py +++ /dev/null @@ -1,312 +0,0 @@ -"""Janitor command output helpers.""" - -from __future__ import annotations - -from datetime import datetime -from typing import TextIO - -from sqlbuild.compiler.migrations.types import OldNameViewDropReason -from sqlbuild.executor.janitor.models import ( - JanitorArchiveCandidate, - JanitorArchivedRelation, - JanitorDeleteCandidate, - JanitorOldNameView, - JanitorOldNameViewPlanning, - JanitorPlan, - JanitorQueryDiffArtifactCandidate, - JanitorSkippedRelation, - JanitorSkippedSchema, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.tree_connector import tree_connector - - -def write_disabled(*, stream: TextIO, use_color: bool = False) -> None: - """Write disabled janitor guidance.""" - - style: CliStyle = CliStyle(use_color=use_color) - disabled_detail: str = ( - "Janitor is opt-in. It previews stale warehouse objects and asks before archiving or " - "deleting anything." - ) - stream.write( - f"{style.warning_strong('Janitor is disabled for this project.')}\n\n" - f"{style.warning(disabled_detail)}\n\n" - f"Add this block to {style.object_name('sqlbuild_project.toml')}:\n\n" - "[janitor]\n" - "enabled = true\n\n" - "After enabling, run janitor again to preview cleanup:\n" - f" {style.accent('sqb janitor')}\n" - ) - - -def _write_plan_summary(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) -> None: - if plan.retention_days == 0: - stream.write(f" {'retention':<22} {style.accent('disabled (0 days)')}\n") - stream.write(f" {'age metadata':<22} {style.accent('not checked')}\n") - else: - stream.write(f" {'retention':<22} {style.accent(f'{plan.retention_days} days')}\n") - if not plan.age_metadata_supported: - stream.write(f" {'age metadata':<22} {style.accent('unavailable')}\n") - if plan.direct_mode: - archive_retention: str = f"{plan.archive_retention_days} days" - stream.write(f" {'archive retention':<22} {style.accent(archive_retention)}\n") - stream.write(f" {'schemas scanned':<22} {style.accent(str(plan.scanned_schema_count))}\n") - stream.write(f" {'schemas skipped':<22} {style.accent(str(len(plan.skipped_schemas)))}\n") - if plan.direct_mode: - _write_count_row( - stream=stream, - style=style, - label="relations to archive", - items=plan.archive_candidates, - ) - _write_count_row( - stream=stream, - style=style, - label="archives to delete", - items=plan.archive_deletion_candidates, - ) - stream.write( - f" {'archives retained':<22} {style.accent(str(len(plan.retained_archives)))}\n" - ) - stream.write( - f" {'old name views':<22} {style.accent(str(len(plan.old_name_views.live)))}\n" - ) - _write_count_row( - stream=stream, - style=style, - label="old name views to drop", - items=plan.old_name_views.drops, - ) - else: - _write_count_row( - stream=stream, - style=style, - label="eligible for deletion", - items=plan.candidates, - ) - _write_count_row( - stream=stream, - style=style, - label="query artifacts", - items=plan.query_diff_artifact_candidates, - ) - _write_count_row( - stream=stream, - style=style, - label="direct state pruned", - items=plan.direct_state_prune_candidates, - ) - skipped_count: str = style.accent(str(len(plan.skipped_relations))) - stream.write(f" {'objects skipped':<22} {skipped_count}\n") - - -def _write_count_row( - *, stream: TextIO, style: CliStyle, label: str, items: tuple[object, ...] -) -> None: - count: str = str(len(items)) - rendered: str = style.warning(count) if items else style.accent(count) - stream.write(f" {label:<22} {rendered}\n") - - -def _write_query_artifact_candidates(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) -> None: - if not plan.query_diff_artifact_candidates: - return - stream.write(f"\n{style.success('Eligible expired query artifacts')}\n") - query_artifact: JanitorQueryDiffArtifactCandidate - for query_artifact in plan.query_diff_artifact_candidates: - stream.write(f" {style.object_name(query_artifact.key.display_name())}\n") - - -def _write_archive_sections(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) -> None: - if plan.archive_candidates: - stream.write(f"\n{style.success('Relations to archive')}\n") - archive_candidate: JanitorArchiveCandidate - for archive_candidate in plan.archive_candidates: - expiry: str = ( - "deleted in this run" - if archive_candidate.expires_at <= archive_candidate.archived_at - else f"delete after {_utc(archive_candidate.expires_at)}" - ) - detail: str = f"{_age(value=archive_candidate.age_timestamp, plan=plan)}, {expiry}" - stream.write( - f" {style.object_name(archive_candidate.key.display_name())} -> " - f"{style.object_name(archive_candidate.archive_key.display_name())} " - f"{style.muted(detail)}\n" - ) - if plan.archive_deletion_candidates: - stream.write(f"\n{style.success('Archives to delete')}\n") - deletion: JanitorArchivedRelation - for deletion in plan.archive_deletion_candidates: - detail = ( - "archived in this run" - if deletion.original_key is not None - else f"archived {_utc(deletion.archived_at)}, " - f"{_age(value=deletion.archived_at, plan=plan)}, " - f"expired {_utc(deletion.expires_at)}" - ) - stream.write( - f" {style.object_name(deletion.key.display_name())} {style.muted(detail)}\n" - ) - if plan.retained_archives: - stream.write(f"\n{style.success('Retained archives')}\n") - retained: JanitorArchivedRelation - for retained in plan.retained_archives: - detail = f"archived {_utc(retained.archived_at)}, expires {_utc(retained.expires_at)}" - stream.write( - f" {style.object_name(retained.key.display_name())} {style.muted(detail)}\n" - ) - - -def _write_old_name_views(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) -> None: - views: JanitorOldNameViewPlanning = plan.old_name_views - listed: tuple[JanitorOldNameView, ...] = (*views.drops, *views.live, *views.missing) - if not listed: - return - stream.write(f"\n{style.success('Old name views')} {style.muted(f'({len(listed)})')}\n") - index: int - view: JanitorOldNameView - for index, view in enumerate(listed): - last: bool = index == len(listed) - 1 - stream.write( - f"{tree_connector(style=style, last=last)} " - f"{style.object_name(view.key.display_name())} " - f"{style.muted(f'-> model:{view.destination_model}')}\n" - ) - label, detail = _old_name_view_detail(view) - stream.write( - f"{' ' if last else style.muted('│ ')}{tree_connector(style=style, last=True)} " - f"{style.muted(label)} {detail}\n" - ) - - -def _old_name_view_detail(view: JanitorOldNameView) -> tuple[str, str]: - expiry: str = "" if view.expires_at is None else f"{view.expires_at:%Y-%m-%d}" - name: str = view.key.display_name() - if view.drop_reason == OldNameViewDropReason.EARLY: - return "drop", f"now (requested; would expire {expiry})" - if view.drop_reason == OldNameViewDropReason.EXPIRED: - return "drop", f"now (expired {expiry})" - if view.occupied: - owner: str = "" if view.claimed_by is None else f", model:{view.claimed_by}" - return "record", f"dropped (name now used by another relation{owner})" - if view.drop_reason == OldNameViewDropReason.MISSING: - return "record", "dropped (the view no longer exists)" - return "expires", f"{expiry} (live; drop early with --drop-old-name-view {name})" - - -def _utc(value: datetime) -> str: - return f"{value:%Y-%m-%d %H:%M:%S} UTC" - - -def _age(*, value: datetime | None, plan: JanitorPlan) -> str: - if value is None or plan.planned_at is None: - return "age unknown" - return f"age {max((plan.planned_at - value).days, 0)}d" - - -def _write_blocked_schemas(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) -> None: - if not plan.blocked_schemas: - return - stream.write(f"\n{style.error_strong('Janitor blocked')}\n") - stream.write(f" {style.error('Managed target schemas contain active configured sources.')}\n") - suppressed_action: str = "archive" if plan.direct_mode else "deletion" - for blocked_schema in plan.blocked_schemas: - sources: str = ", ".join(blocked_schema.source_names) - stream.write( - f" {style.object_name(blocked_schema.display_name())} " - f"{style.error_muted('active sources: ' + sources)}\n" - ) - for candidate in blocked_schema.suppressed_candidates: - suppressed: str = f"suppressed {suppressed_action}: {candidate.key.display_name()}" - stream.write(f" {style.error_muted(suppressed)}\n") - for archived in blocked_schema.suppressed_archive_deletions: - suppressed = f"suppressed archive deletion: {archived.key.display_name()}" - stream.write(f" {style.error_muted(suppressed)}\n") - stream.write(f" {style.error('No janitor actions will be performed.')}\n") - - -def write_plan(*, plan: JanitorPlan, stream: TextIO, use_color: bool = False) -> None: - """Write a janitor preview.""" - - style: CliStyle = CliStyle(use_color=use_color) - env_label: str = environment_label(plan) - rendered_env: str = style.object_name(env_label) - stream.write(f"{style.title('Janitor preview')} {rendered_env}\n\n") - _write_plan_summary(plan=plan, stream=stream, style=style) - - _write_blocked_schemas(plan=plan, stream=stream, style=style) - - if plan.skipped_schemas: - stream.write(f"\n{style.success('Skipped schemas')}\n") - skipped_schema: JanitorSkippedSchema - for skipped_schema in plan.skipped_schemas: - sources: str = ", ".join(skipped_schema.source_names) - stream.write( - f" {style.object_name(skipped_schema.display_name())} " - f"{style.muted('contains active source ' + sources)}\n" - ) - - if plan.candidates and not plan.direct_mode: - stream.write(f"\n{style.success('Eligible objects')}\n") - candidate: JanitorDeleteCandidate - for candidate in plan.candidates: - stream.write(f" {style.object_name(candidate.key.display_name())}\n") - - _write_archive_sections(plan=plan, stream=stream, style=style) - - _write_old_name_views(plan=plan, stream=stream, style=style) - - _write_query_artifact_candidates(plan=plan, stream=stream, style=style) - - if plan.direct_state_prune_candidates: - stream.write(f"\n{style.success('Eligible direct state pruning')}\n") - for direct_state_candidate in plan.direct_state_prune_candidates: - stream.write( - f" {style.object_name(direct_state_candidate.display_name())} " - f"{style.muted(f'keep latest {direct_state_candidate.retain_versions}')}\n" - ) - - if plan.skipped_relations: - stream.write(f"\n{style.success('Skipped objects')}\n") - skipped: JanitorSkippedRelation - for skipped in plan.skipped_relations: - stream.write( - f" {style.object_name(skipped.key.display_name())} " - f"{style.muted(skipped.reason)}\n" - ) - stream.write("\n") - - -def confirmation_text(plan: JanitorPlan) -> str: - """Build the exact confirmation phrase for a janitor plan.""" - - state_candidate_count: int = len(plan.direct_state_prune_candidates) - archive_prefix: str = ( - f"archive {len(plan.archive_candidates)} and " if plan.archive_candidates else "" - ) - physical_deletion_count: int = physical_janitor_deletion_count(plan) - if state_candidate_count == 0: - return ( - f"{archive_prefix}delete {physical_deletion_count} objects " - f"from {environment_label(plan)}" - ) - deletion_count: int = physical_deletion_count + state_candidate_count - return f"{archive_prefix}delete {deletion_count} items from {environment_label(plan)}" - - -def physical_janitor_deletion_count(plan: JanitorPlan) -> int: - """Count warehouse relations the janitor plan will drop.""" - - relation_count: int = ( - len(plan.archive_deletion_candidates) if plan.direct_mode else len(plan.candidates) - ) - return ( - relation_count + len(plan.query_diff_artifact_candidates) + len(plan.old_name_views.drops) - ) - - -def environment_label(plan: JanitorPlan) -> str: - """Render the janitor environment label.""" - - return plan.target_name if plan.target_name is not None else "default" diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py b/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py deleted file mode 100644 index db795f3185..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Janitor command output phases.""" - -from __future__ import annotations - -import sys - -from sqlbuild.cli.commands._helpers.janitor_output.output import ( - confirmation_text, - environment_label, - physical_janitor_deletion_count, - write_disabled, - write_plan, -) -from sqlbuild.cli.commands.models import ( - JanitorInvocation, - JanitorPlanningResult, -) -from sqlbuild.executor.janitor.models import JanitorExecutionResult, JanitorPlan -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.count_noun import format_count_noun - - -def write_janitor_disabled(*, invocation: JanitorInvocation) -> None: - """Write janitor disabled output.""" - - write_disabled(stream=sys.stdout, use_color=invocation.use_color) - - -def write_janitor_preview_start(*, invocation: JanitorInvocation) -> None: - """Announce an inspection-only ``--as`` janitor preview before any work starts.""" - - sys.stdout.write( - f"Previewing janitor as target '{invocation.as_target}' through " - f"{_active_connection_label(invocation=invocation)} (inspection only).\n" - ) - sys.stdout.flush() - - -def write_janitor_preview_complete(*, invocation: JanitorInvocation) -> None: - """Close an ``--as`` janitor preview with an unambiguous nothing-changed state.""" - - style: CliStyle = CliStyle(use_color=invocation.use_color) - sys.stdout.write( - style.success( - f"Previewed janitor as target '{invocation.as_target}' through " - f"{_active_connection_label(invocation=invocation)}. Nothing was changed." - ) - + "\n" - ) - sys.stdout.write( - f"Rerun with `--target {invocation.as_target}` instead of " - f"`--as {invocation.as_target}` to apply it.\n" - ) - - -def _active_connection_label(*, invocation: JanitorInvocation) -> str: - if invocation.active_target_name is None: - return "the active connection" - return f"the connection of target '{invocation.active_target_name}'" - - -def write_janitor_plan( - *, invocation: JanitorInvocation, planning_result: JanitorPlanningResult -) -> None: - """Write janitor plan preview output.""" - - write_plan(plan=planning_result.plan, stream=sys.stdout, use_color=invocation.use_color) - - -def janitor_plan_has_work(planning_result: JanitorPlanningResult) -> bool: - """Return whether the janitor plan has any cleanup work.""" - - plan: JanitorPlan = planning_result.plan - return bool( - (plan.candidates and not plan.direct_mode) - or plan.archive_candidates - or plan.archive_deletion_candidates - or plan.query_diff_artifact_candidates - or plan.direct_state_prune_candidates - or plan.old_name_views.drops - or plan.old_name_views.missing - ) - - -def confirm_janitor_plan(*, planning_result: JanitorPlanningResult) -> bool: - """Prompt for janitor confirmation.""" - - plan: JanitorPlan = planning_result.plan - expected: str = confirmation_text(plan) - prune_count: int = len(plan.direct_state_prune_candidates) - archive_prefix: str = ( - f"archive {len(plan.archive_candidates)} and " if plan.archive_candidates else "" - ) - if prune_count: - physical_deletion_count: int = physical_janitor_deletion_count(plan) - deletion_count: int = physical_deletion_count + prune_count - sys.stdout.write( - f"Janitor will {archive_prefix}delete {deletion_count} items " - f"from {environment_label(plan)}.\n" - ) - else: - object_count: int = ( - physical_janitor_deletion_count(plan) - if plan.direct_mode - else len(plan.candidates) + len(plan.query_diff_artifact_candidates) - ) - sys.stdout.write( - f"Janitor will {archive_prefix}delete {object_count} objects " - f"from {environment_label(plan)}.\n" - ) - if plan.retention_days == 0: - sys.stdout.write("Retention: disabled (0 days)\n") - sys.stdout.write("Age metadata will not be checked.\n") - else: - sys.stdout.write(f"Retention: {plan.retention_days} days\n") - if plan.direct_mode: - sys.stdout.write(f"Archive retention: {plan.archive_retention_days} days\n") - sys.stdout.write(f"\nType `{expected}` to continue: ") - sys.stdout.flush() - try: - response: str = input() - except KeyboardInterrupt: - sys.stdout.write("\n") - return False - return response == expected - - -def write_janitor_cancelled() -> None: - """Write janitor cancellation output.""" - - sys.stdout.write("Janitor cancelled.\n") - - -def write_janitor_completion( - *, invocation: JanitorInvocation, result: JanitorExecutionResult -) -> None: - """Write janitor execution summary.""" - - style: CliStyle = CliStyle(use_color=invocation.use_color) - sys.stdout.write(style.success(_deleted_message(result=result)) + "\n") - - -def _deleted_message(*, result: JanitorExecutionResult) -> str: - archived_prefix: str = ( - f"Archived {format_count_noun(count=len(result.archived), singular='relation')}. " - if result.archived - else "" - ) - return archived_prefix + _deletion_summary(result=result) - - -def _deletion_summary(*, result: JanitorExecutionResult) -> str: - deleted_object_count: int = ( - len(result.deleted) - + len(result.deleted_archives) - + len(result.deleted_query_diff_artifacts) - + len(result.dropped_old_name_views) - ) - objects: str = format_count_noun(count=deleted_object_count, singular="object") - pruned_state_count: int = len(result.pruned_direct_state) - if pruned_state_count: - pruned: str = format_count_noun(count=pruned_state_count, singular="direct state table") - return f"Deleted {objects} and pruned {pruned}." - return f"Deleted {objects}." diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py index b23675a53f..17f5b4df22 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py @@ -5,7 +5,7 @@ import sys import time -from sqlbuild._native import build_compiled_project +from sqlbuild._native import PlanningProgressReporter, build_compiled_project from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) @@ -14,7 +14,6 @@ JanitorCompileContext, JanitorInvocation, ) -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py index f1e40e2a6a..0c9e5f71f9 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py @@ -5,12 +5,12 @@ import sys import time +from sqlbuild._native import ConnectionProgressReporter from sqlbuild.cli.commands.models import ( JanitorCompileContext, JanitorConnectionContext, JanitorInvocation, ) -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py deleted file mode 100644 index 5264a0bf7b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Janitor command invocation and settings phases.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import discover_project_inputs -from sqlbuild.cli.commands._helpers.runtime.preview_target import validate_preview_target -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - JanitorCommandRequest, - JanitorInvocation, - JanitorSettings, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.janitor.main.resolve_retention_days import resolve_janitor_retention_days -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name - - -def resolve_janitor_invocation(*, request: JanitorCommandRequest) -> JanitorInvocation: - """Resolve project discovery and output context for janitor.""" - - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - validate_preview_target( - discovered_inputs=discovered_inputs, as_target=request.as_target, command_name="janitor" - ) - active_target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=request.selected_target, - ) - return JanitorInvocation( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - use_color=not request.no_color and supports_color(), - selected_target=request.selected_target, - active_target_name=active_target_name, - as_target=request.as_target, - ) - - -def resolve_janitor_settings( - *, request: JanitorCommandRequest, invocation: JanitorInvocation -) -> JanitorSettings: - """Resolve and validate effective janitor settings.""" - - retention_days: int = resolve_janitor_retention_days( - override=request.retention_days, - configured=invocation.discovered_inputs.project_config.janitor.retention_days, - ) - if retention_days < 0: - raise CliUserError("janitor --retention-days must be >= 0", code="C501") - direct_state_history_versions: int = ( - request.direct_state_history_versions - if request.direct_state_history_versions is not None - else invocation.discovered_inputs.project_config.janitor.direct_state_history_versions - ) - if direct_state_history_versions < 0: - raise CliUserError("janitor --direct-state-history-versions must be >= 0", code="C502") - return JanitorSettings( - retention_days=retention_days, - direct_state_history_versions=direct_state_history_versions, - archive_retention_days=( - invocation.discovered_inputs.project_config.janitor.archive_retention_days - ), - drop_old_name_views=request.drop_old_name_views, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/__init__.py b/src/sqlbuild/cli/commands/_helpers/lineage/__init__.py deleted file mode 100644 index e43f8a4af5..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/lineage/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Lineage command helpers.""" diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/cache.py b/src/sqlbuild/cli/commands/_helpers/lineage/cache.py deleted file mode 100644 index 62ee8c8c51..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/lineage/cache.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Disposable SQLite cache for relation-level project lineage.""" - -from __future__ import annotations - -import json -import os -import sqlite3 -import sys -import tempfile -from contextlib import closing -from importlib.metadata import PackageNotFoundError, version -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.cli.commands.constants import TARGET_DIRECTORY_NAME -from sqlbuild.cli.commands.models import LineageNode, RelationLineageIndex -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.frontier.constants import CACHE_DIRECTORY_NAME, NATIVE_CACHE_NAMESPACE_SUFFIX -from sqlbuild.compiler.pipeline.models import ProjectGraph - -_CACHE_SCHEMA_VERSION: int = 1 -_CACHE_ALGORITHM_VERSION: str = "relation-lineage-graph-v4" -_LINEAGE_CACHE_DIRECTORY_NAME: str = "lineage" -_CACHE_FILE_RELATIVE_PATH: Path = Path("v1/structural-graph.sqlite3") -_CACHE_MAX_BYTES: int = 50_000_000 -_CACHE_MAX_ROWS: int = 1_000_000 -_SQLITE_TIMEOUT_SECONDS: float = 5.0 -_FIRST_PYTHON_SKIPPING_INTERRUPTED_LISTINGS: tuple[int, int] = (3, 13) -_INTERRUPTED_LISTING_UNCACHEABLE: bool = ( - sys.version_info < _FIRST_PYTHON_SKIPPING_INTERRUPTED_LISTINGS -) - - -def relation_lineage_fingerprint( - *, project_dir: Path, cli_vars: dict[str, object] | None -) -> str | None: - """Hash authored project inputs and invocation context without retaining their values.""" - - try: - prefix: bytes = _fingerprint_prefix(cli_vars=cli_vars) - except (TypeError, UnicodeError, ValueError): - return None - return _native.relation_lineage_fingerprint( - project_dir, prefix, _INTERRUPTED_LISTING_UNCACHEABLE - ) - - -def _fingerprint_prefix(*, cli_vars: dict[str, object] | None) -> bytes: - return ( - _CACHE_ALGORITHM_VERSION.encode("ascii") - + _sqlbuild_version().encode("utf-8") - + json.dumps( - {} if cli_vars is None else cli_vars, - ensure_ascii=True, - separators=(",", ":"), - sort_keys=True, - ).encode("ascii") - ) - - -def read_relation_lineage_cache( - *, project_dir: Path, fingerprint: str -) -> RelationLineageIndex | None: - """Read a verified structural graph, treating every fault as a cache miss.""" - - database_path: Path = _cache_path(project_dir=project_dir) - try: - if database_path.stat().st_size > _CACHE_MAX_BYTES: - return None - with closing( - sqlite3.connect( - f"file:{database_path}?mode=ro", - uri=True, - timeout=_SQLITE_TIMEOUT_SECONDS, - ) - ) as connection: - metadata: dict[str, str] = dict(connection.execute("SELECT key, value FROM metadata")) - if metadata != { - "algorithm_version": _CACHE_ALGORITHM_VERSION, - "input_fingerprint": fingerprint, - "schema_version": str(_CACHE_SCHEMA_VERSION), - }: - return None - return _read_index(connection=connection) - except (KeyError, OSError, sqlite3.DatabaseError, TypeError, ValueError): - return None - - -def write_relation_lineage_cache( - *, project_dir: Path, fingerprint: str, graph: ProjectGraph -) -> RelationLineageIndex: - """Build the structural projection and atomically publish its SQLite cache.""" - - index: RelationLineageIndex = _relation_index(graph=graph) - directory: Path = _cache_path(project_dir=project_dir).parent - temporary_path: Path | None = None - try: - directory.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp(prefix=".structural-graph-", dir=directory) - os.close(descriptor) - temporary_path = Path(temporary_name) - with closing( - sqlite3.connect(temporary_path, timeout=_SQLITE_TIMEOUT_SECONDS) - ) as connection: - _create_schema(connection=connection) - _write_index(connection=connection, fingerprint=fingerprint, index=index) - connection.commit() - temporary_path.replace(_cache_path(project_dir=project_dir)) - except (OSError, sqlite3.DatabaseError, TypeError, ValueError): - pass - finally: - if temporary_path is not None: - temporary_path.unlink(missing_ok=True) - return index - - -def _cache_path(*, project_dir: Path) -> Path: - return ( - project_dir - / TARGET_DIRECTORY_NAME - / CACHE_DIRECTORY_NAME - / f"{_LINEAGE_CACHE_DIRECTORY_NAME}{NATIVE_CACHE_NAMESPACE_SUFFIX}" - / _CACHE_FILE_RELATIVE_PATH - ) - - -def _relation_index(*, graph: ProjectGraph) -> RelationLineageIndex: - keys: frozenset[CompiledObjectKey] = frozenset(graph.all_keys.values()) - return RelationLineageIndex( - nodes={key: _lineage_node(project=graph.project, key=key) for key in keys}, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, - all_keys=graph.all_keys, - ) - - -def _lineage_node(*, project: CompiledProjectView, key: CompiledObjectKey) -> LineageNode: - for model in project.models: - if model.key == key: - return LineageNode( - key=key, - relative_path=str(model.relative_path), - qualified_name=model.destination.qualified_name, - ) - for seed in project.seeds: - if seed.key == key: - return LineageNode( - key=key, - relative_path=str(seed.seed_file.relative_path), - qualified_name=seed.destination.qualified_name, - ) - for source in project.sources: - if source.key == key: - return LineageNode( - key=key, - relative_path=str(source.source_file.relative_path), - qualified_name=_source_relation_name(source.source_entry), - ) - return LineageNode(key=key) - - -def _source_relation_name(source: object) -> str | None: - database: str | None = getattr(source, "database", None) - schema: str | None = getattr(source, "schema", None) - table: str | None = getattr(source, "table", None) - if table is None: - return None - if database is not None and schema is not None: - return f"{database}.{schema}.{table}" - if schema is not None: - return f"{schema}.{table}" - return table - - -def _create_schema(*, connection: sqlite3.Connection) -> None: - connection.executescript( - """ - CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL); - CREATE TABLE resources ( - resource_type TEXT NOT NULL, - resource_name TEXT NOT NULL, - relative_path TEXT, - qualified_name TEXT, - PRIMARY KEY (resource_type, resource_name) - ); - CREATE TABLE lookups ( - lookup_name TEXT PRIMARY KEY, - resource_type TEXT NOT NULL, - resource_name TEXT NOT NULL - ); - CREATE TABLE edges ( - upstream_type TEXT NOT NULL, - upstream_name TEXT NOT NULL, - downstream_type TEXT NOT NULL, - downstream_name TEXT NOT NULL, - PRIMARY KEY (upstream_type, upstream_name, downstream_type, downstream_name) - ); - CREATE INDEX edges_downstream ON edges (downstream_type, downstream_name); - CREATE INDEX edges_upstream ON edges (upstream_type, upstream_name); - CREATE TABLE tags ( - tag TEXT NOT NULL, - resource_type TEXT NOT NULL, - resource_name TEXT NOT NULL, - PRIMARY KEY (tag, resource_type, resource_name) - ); - CREATE TABLE paths ( - resource_type TEXT NOT NULL, - resource_name TEXT NOT NULL, - folder TEXT NOT NULL, - PRIMARY KEY (resource_type, resource_name) - ); - """ - ) - - -def _write_index( - *, connection: sqlite3.Connection, fingerprint: str, index: RelationLineageIndex -) -> None: - connection.executemany( - "INSERT INTO metadata (key, value) VALUES (?, ?)", - ( - ("algorithm_version", _CACHE_ALGORITHM_VERSION), - ("input_fingerprint", fingerprint), - ("schema_version", str(_CACHE_SCHEMA_VERSION)), - ), - ) - connection.executemany( - "INSERT INTO resources VALUES (?, ?, ?, ?)", - ( - (str(key.resource_type), key.name, node.relative_path, node.qualified_name) - for key, node in index.nodes.items() - ), - ) - connection.executemany( - "INSERT INTO lookups VALUES (?, ?, ?)", - ((name, str(key.resource_type), key.name) for name, key in index.all_keys.items()), - ) - connection.executemany( - "INSERT INTO edges VALUES (?, ?, ?, ?)", - _edge_rows(index=index), - ) - connection.executemany( - "INSERT INTO tags VALUES (?, ?, ?)", - _tag_rows(index=index), - ) - connection.executemany( - "INSERT INTO paths VALUES (?, ?, ?)", - ((str(key.resource_type), key.name, folder) for key, folder in index.path_index.items()), - ) - - -def _edge_rows(*, index: RelationLineageIndex) -> list[tuple[str, str, str, str]]: - rows: list[tuple[str, str, str, str]] = [] - for downstream, upstreams in index.upstream_deps.items(): - for upstream in upstreams: - rows.append( - ( - str(upstream.resource_type), - upstream.name, - str(downstream.resource_type), - downstream.name, - ) - ) - return rows - - -def _tag_rows(*, index: RelationLineageIndex) -> list[tuple[str, str, str]]: - rows: list[tuple[str, str, str]] = [] - for tag, keys in index.tag_index.items(): - for key in keys: - rows.append((tag, str(key.resource_type), key.name)) - return rows - - -def _read_index(*, connection: sqlite3.Connection) -> RelationLineageIndex | None: - resource_rows: list[tuple[str, str, str | None, str | None]] = connection.execute( - "SELECT resource_type, resource_name, relative_path, qualified_name FROM resources" - ).fetchall() - edge_rows: list[tuple[str, str, str, str]] = connection.execute( - "SELECT upstream_type, upstream_name, downstream_type, downstream_name FROM edges" - ).fetchall() - lookup_rows: list[tuple[str, str, str]] = connection.execute( - "SELECT lookup_name, resource_type, resource_name FROM lookups" - ).fetchall() - tag_rows: list[tuple[str, str, str]] = connection.execute( - "SELECT tag, resource_type, resource_name FROM tags" - ).fetchall() - path_rows: list[tuple[str, str, str]] = connection.execute( - "SELECT resource_type, resource_name, folder FROM paths" - ).fetchall() - if ( - sum(map(len, (resource_rows, edge_rows, lookup_rows, tag_rows, path_rows))) - > _CACHE_MAX_ROWS - ): - return None - keys: dict[tuple[str, str], CompiledObjectKey] = { - (resource_type, name): CompiledObjectKey(resource_type=resource_type, name=name) - for resource_type, name, _, _ in resource_rows - } - nodes: dict[CompiledObjectKey, LineageNode] = { - keys[(resource_type, name)]: LineageNode( - key=keys[(resource_type, name)], - relative_path=relative_path, - qualified_name=qualified_name, - ) - for resource_type, name, relative_path, qualified_name in resource_rows - } - upstream_mutable: dict[CompiledObjectKey, list[CompiledObjectKey]] = {} - downstream_mutable: dict[CompiledObjectKey, list[CompiledObjectKey]] = {} - for upstream_type, upstream_name, downstream_type, downstream_name in edge_rows: - upstream: CompiledObjectKey = keys[(upstream_type, upstream_name)] - downstream: CompiledObjectKey = keys[(downstream_type, downstream_name)] - upstream_mutable.setdefault(downstream, []).append(upstream) - downstream_mutable.setdefault(upstream, []).append(downstream) - tags_mutable: dict[str, set[CompiledObjectKey]] = {} - for tag, resource_type, name in tag_rows: - tags_mutable.setdefault(tag, set()).add(keys[(resource_type, name)]) - return RelationLineageIndex( - nodes=nodes, - upstream_deps={key: tuple(value) for key, value in upstream_mutable.items()}, - downstream_deps={key: tuple(value) for key, value in downstream_mutable.items()}, - tag_index={tag: frozenset(value) for tag, value in tags_mutable.items()}, - path_index={ - keys[(resource_type, name)]: folder for resource_type, name, folder in path_rows - }, - all_keys={ - lookup_name: keys[(resource_type, name)] - for lookup_name, resource_type, name in lookup_rows - }, - ) - - -def _sqlbuild_version() -> str: - try: - return version("sqlbuild") - except PackageNotFoundError: - return "0+unknown" diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py deleted file mode 100644 index 5a25c8072b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py +++ /dev/null @@ -1,307 +0,0 @@ -"""Lineage command preparation, selection, and rendering.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.lineage.cache import ( - read_relation_lineage_cache, - relation_lineage_fingerprint, - write_relation_lineage_cache, -) -from sqlbuild.cli.commands._helpers.lineage.output import ( - format_column_lineage_json, - format_column_lineage_list, - format_column_lineage_tree, - format_lineage_json, - format_lineage_list, - format_lineage_tree, -) -from sqlbuild.cli.commands._helpers.lineage.selection import ( - normalize_lineage_target, - parse_depth, - select_column_target_lineage, - select_selector_lineage, - select_target_lineage, -) -from sqlbuild.cli.commands.constants import ( - COLUMN_TARGET_SEPARATOR, - JSON_OUTPUT_FORMAT, - LIST_OUTPUT_FORMAT, - UPSTREAM_DIRECTION, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - ColumnLineageTrace, - LineageCommandRequest, - LineageGraph, - RelationLineageIndex, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.lineage.models import DirectSemanticColumnUse -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.constants import UNKNOWN_SELECTOR_ERROR_CODE -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) - - -def execute_lineage(*, request: LineageCommandRequest) -> int: - """Execute a validated lineage request through focused phases.""" - - _validate_request(request=request) - graph, adapter, parsed_depth = _prepare_graph(request=request) - uses: tuple[DirectSemanticColumnUse, ...] = _semantic_uses( - request=request, graph=graph, adapter=adapter - ) - rendered: str = _render_lineage(request=request, graph=graph, parsed_depth=parsed_depth) - _write_lineage( - rendered=rendered, - output_format=request.output_format, - uses=uses, - include_uses=request.include_uses, - ) - return 0 - - -def _validate_request(*, request: LineageCommandRequest) -> None: - if request.targets and request.select: - raise CliUserError("lineage accepts either targets or --select, not both", code="C301") - if not request.targets and not request.select: - raise CliUserError("lineage requires a target or --select", code="C302") - if len(request.targets) > 1 and any( - COLUMN_TARGET_SEPARATOR in target for target in request.targets - ): - raise CliUserError( - "column lineage accepts exactly one model.column target", - code="C320", - help="run one lineage command per column", - ) - if request.exclude and not request.select: - raise CliUserError("--exclude can only be used with --select", code="C303") - - -def _prepare_graph( - *, request: LineageCommandRequest -) -> tuple[ProjectGraph | RelationLineageIndex, BaseAdapter | None, int | None]: - parsed_depth: int | None = parse_depth(request.depth) - project_dir: Path = request.project_dir or Path.cwd() - requires_compiled_graph: bool = _requires_compiled_graph(request=request) - fingerprint: str | None = ( - None - if requires_compiled_graph - else relation_lineage_fingerprint(project_dir=project_dir, cli_vars=request.cli_vars) - ) - if fingerprint is not None: - cached: RelationLineageIndex | None = read_relation_lineage_cache( - project_dir=project_dir, - fingerprint=fingerprint, - ) - if cached is not None: - return cached, None, parsed_depth - from sqlbuild._native import build_project_graph, discover_project_inputs - from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter - - discovered: DiscoveredProjectInputs = discover_project_inputs( - project_dir=project_dir, - sql_analysis_enabled_override=(None if requires_compiled_graph else False), - extract_output_column_locations=False, - ) - reject_unit_test_selectors( - select=request.select, exclude=request.exclude, discovered_inputs=discovered - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=resolve_effective_adapter_name( - project_config=discovered.project_config, - local_config=discovered.local_config, - ), - project_dir=project_dir, - ) - graph: ProjectGraph = build_project_graph( - discovered_inputs=discovered, - adapter=adapter, - no_sql_validation=request.no_sql_validation or not requires_compiled_graph, - skip_column_inference=not requires_compiled_graph, - cli_vars=request.cli_vars, - ) - if fingerprint is not None: - return ( - write_relation_lineage_cache( - project_dir=project_dir, - fingerprint=fingerprint, - graph=graph, - ), - None, - parsed_depth, - ) - return graph, adapter, parsed_depth - - -def _requires_compiled_graph(*, request: LineageCommandRequest) -> bool: - return request.include_uses or any( - COLUMN_TARGET_SEPARATOR in target for target in request.targets - ) - - -def _semantic_uses( - *, - request: LineageCommandRequest, - graph: ProjectGraph | RelationLineageIndex, - adapter: BaseAdapter | None, -) -> tuple[DirectSemanticColumnUse, ...]: - if not request.include_uses: - return () - if not isinstance(graph, ProjectGraph) or adapter is None: - raise CliUserError( - "semantic uses require a compiled project graph", - code="C307", - ) - return _native.build_direct_semantic_uses( - graph.project, - adapter.sql_analysis_dialect(), - _semantic_use_model_names(request=request, graph=graph), - ) - - -def _render_lineage( - *, - request: LineageCommandRequest, - graph: ProjectGraph | RelationLineageIndex, - parsed_depth: int | None, -) -> str: - if request.targets: - targets: tuple[str, ...] = tuple( - normalize_lineage_target(graph=graph, target=target) for target in request.targets - ) - direction: str = request.direction or UPSTREAM_DIRECTION - column_trace: ColumnLineageTrace | None = ( - select_column_target_lineage( - graph=graph, - target=targets[0], - direction=direction, - depth=parsed_depth, - mode=request.lineage_mode, - ) - if len(targets) == 1 - else None - ) - if column_trace is not None: - return _format_column_trace(trace=column_trace, output_format=request.output_format) - lineage_graph: LineageGraph = select_target_lineage( - graph=graph, - targets=targets, - direction=direction, - depth=parsed_depth, - ) - else: - try: - lineage_graph = select_selector_lineage( - graph=graph, - select=request.select, - exclude=request.exclude, - depth=parsed_depth, - direction=request.direction, - ) - except PlannerInputError as error: - if error.code == UNKNOWN_SELECTOR_ERROR_CODE and isinstance( - graph, RelationLineageIndex - ): - from sqlbuild._native import discover_project_inputs - - reject_unit_test_selectors( - select=request.select, - exclude=request.exclude, - discovered_inputs=discover_project_inputs( - project_dir=request.project_dir or Path.cwd(), - sql_analysis_enabled_override=False, - extract_output_column_locations=False, - ), - ) - raise - return _format_graph(graph=lineage_graph, output_format=request.output_format) - - -def _format_column_trace(*, trace: ColumnLineageTrace, output_format: str) -> str: - if output_format == JSON_OUTPUT_FORMAT: - return format_column_lineage_json(trace) - if output_format == LIST_OUTPUT_FORMAT: - return format_column_lineage_list(trace=trace, use_color=supports_color()) - return format_column_lineage_tree(trace=trace, use_color=supports_color()) - - -def _format_graph(*, graph: LineageGraph, output_format: str) -> str: - if output_format == JSON_OUTPUT_FORMAT: - return format_lineage_json(graph) - if output_format == LIST_OUTPUT_FORMAT: - return format_lineage_list(graph=graph, use_color=supports_color()) - return format_lineage_tree(graph=graph, use_color=supports_color()) - - -def _semantic_use_model_names( - *, request: LineageCommandRequest, graph: ProjectGraph -) -> frozenset[str]: - if request.targets: - return frozenset( - normalize_lineage_target(graph=graph, target=target).split( - COLUMN_TARGET_SEPARATOR, maxsplit=1 - )[0] - for target in request.targets - ) - from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph - - keys: frozenset[CompiledObjectKey] = select_project_graph( - graph=graph, select=request.select, exclude=request.exclude - ) - return frozenset(key.name for key in keys if key.resource_type == CompiledResourceType.MODEL) - - -def _write_lineage( - *, - rendered: str, - output_format: str, - uses: tuple[DirectSemanticColumnUse, ...], - include_uses: bool, -) -> None: - if not include_uses: - print(rendered if output_format == JSON_OUTPUT_FORMAT else f"\n{rendered}\n") - return - if output_format == JSON_OUTPUT_FORMAT: - payload: dict[str, object] = json.loads(rendered) - payload["semantic_uses"] = [_serialize_use(use=use) for use in uses] - print(json.dumps(payload, indent=2)) - return - lines: list[str] = ["", rendered, "", "Direct semantic column uses"] - if not uses: - lines.append(" none") - else: - lines.extend( - f" {use.consumer_model} [{use.context}] " - f"{use.source.resource_type}:{use.source.resource_name}.{use.source.column_name} " - f"<- {use.expression_sql}" - for use in uses - ) - print("\n".join(lines) + "\n") - - -def _serialize_use(*, use: DirectSemanticColumnUse) -> dict[str, object]: - return { - "consumer_model": use.consumer_model, - "context": use.context, - "expression": use.expression_sql, - "source": { - "resource_type": str(use.source.resource_type), - "resource_name": use.source.resource_name, - "column": use.source.column_name, - }, - "confidence": use.confidence.value, - "line": use.line, - "column": use.column, - } diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/output.py b/src/sqlbuild/cli/commands/_helpers/lineage/output.py deleted file mode 100644 index b962ec46e1..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/lineage/output.py +++ /dev/null @@ -1,286 +0,0 @@ -"""Lineage output formatting.""" - -from __future__ import annotations - -import json - -from sqlbuild._native import ( - render_column_trace_limit_note, - render_column_trace_tree, - render_dependency_tree, - serialize_column, - serialize_column_edge, -) -from sqlbuild.cli.commands.constants import ( - BOTH_DIRECTIONS, - DOWNSTREAM_DIRECTION, - UPSTREAM_DIRECTION, -) -from sqlbuild.cli.commands.models import ( - ColumnLineageTrace, - LineageGraph, - LineageNode, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.models import ColumnLineageEdge, QualifiedLineageColumn -from sqlbuild.compiler.lineage.types import ColumnTransformKind -from sqlbuild.presentation.classes.cli_style import CliStyle - -_HUMAN_COLUMN_TRACE_LIMIT: int = 25 - - -def format_lineage_json(graph: LineageGraph) -> str: - """Serialize lineage graph as stable JSON.""" - - payload: dict[str, object] = { - "nodes": [_serialize_node(node) for node in graph.nodes], - "edges": [ - {"from": _node_id(upstream), "to": _node_id(downstream)} - for upstream, downstream in graph.edges - ], - } - if graph.focus_keys: - payload["focus"] = [_node_id(key) for key in graph.focus_keys] - if graph.direction is not None: - payload["direction"] = graph.direction - return json.dumps(payload, indent=2) - - -def format_column_lineage_json(trace: ColumnLineageTrace) -> str: - """Serialize column lineage trace as stable JSON.""" - - payload: dict[str, object] = { - "target": serialize_column(column=trace.target, render_resource_type=_render_resource_type), - "direction": trace.direction, - "metadata": { - "mode": trace.mode.value, - "max_depth": trace.max_depth, - "analyzed_models": trace.analyzed_model_count, - "truncated": trace.truncated, - }, - "trace": [ - serialize_column_edge(edge=edge, render_resource_type=_render_resource_type) - for edge in trace.trace - ], - } - return json.dumps(payload, indent=2) - - -def format_lineage_list(*, graph: LineageGraph, use_color: bool = True) -> str: - """Format lineage graph as an edge list.""" - - style: CliStyle = CliStyle(use_color=use_color) - if not graph.edges: - return "\n".join(_format_key(key=node.key, style=style) for node in graph.nodes) - left_width: int = max(len(_node_id(upstream)) for upstream, _downstream in graph.edges) - return "\n".join( - f"{_format_key(key=upstream, style=style)}" - f"{' ' * (left_width - len(_node_id(upstream)))} " - f"{style.muted('->')} " - f"{_format_key(key=downstream, style=style)}" - for upstream, downstream in graph.edges - ) - - -def format_column_lineage_list( - *, - trace: ColumnLineageTrace, - use_color: bool = True, -) -> str: - """Format column lineage as a flat dependency list.""" - - style: CliStyle = CliStyle(use_color=use_color) - if not trace.trace: - return f"Column dependencies\n\n{_format_column(column=trace.target, style=style)}" - displayed_trace: tuple[ColumnLineageEdge, ...] = trace.trace[:_HUMAN_COLUMN_TRACE_LIMIT] - left_width: int = max(len(_column_id(edge.source)) for edge in displayed_trace) - right_width: int = max(len(_column_id(edge.target)) for edge in displayed_trace) - lines: list[str] = [style.title("Column dependencies"), ""] - lines.extend( - f"{_format_column(column=edge.source, style=style)}" - f"{' ' * (left_width - len(_column_id(edge.source)))} " - f"{style.muted('->')} " - f"{_format_column(column=edge.target, style=style)}" - f"{' ' * (right_width - len(_column_id(edge.target)))} " - f"{_format_transform(edge=edge, style=style)}" - for edge in displayed_trace - ) - lines.extend( - render_column_trace_limit_note( - total=len(trace.trace), - limit=_HUMAN_COLUMN_TRACE_LIMIT, - note_style=style.muted, - ) - ) - return "\n".join(lines) - - -def format_lineage_tree(*, graph: LineageGraph, use_color: bool = True) -> str: - """Format lineage graph for humans.""" - - if len(graph.focus_keys) != 1 or graph.direction is None: - return _format_graph_summary(graph=graph, use_color=use_color) - style: CliStyle = CliStyle(use_color=use_color) - focus: CompiledObjectKey = graph.focus_keys[0] - node_by_key: dict[CompiledObjectKey, LineageNode] = {node.key: node for node in graph.nodes} - title: str = style.title("Lineage") - direction: str = style.muted(graph.direction) - lines: list[str] = [ - f"{title} {_format_node(node=node_by_key[focus], style=style)} {direction}" - ] - if graph.direction in {UPSTREAM_DIRECTION, BOTH_DIRECTIONS}: - upstream: dict[CompiledObjectKey, list[CompiledObjectKey]] = {} - for parent, child in graph.edges: - upstream.setdefault(child, []).append(parent) - if graph.direction == BOTH_DIRECTIONS: - lines.append(style.section("upstream")) - lines.extend( - render_dependency_tree( - focus=focus, - deps=upstream, - seen={focus}, - format_node=lambda key: _format_node(node=node_by_key[key], style=style), - sort_key=lambda key: (str(key.resource_type), key.name), - branch_style=style.muted, - already_shown=lambda: style.muted(" (already shown)"), - ) - ) - if graph.direction in {DOWNSTREAM_DIRECTION, BOTH_DIRECTIONS}: - downstream: dict[CompiledObjectKey, list[CompiledObjectKey]] = {} - for parent, child in graph.edges: - downstream.setdefault(parent, []).append(child) - if graph.direction == BOTH_DIRECTIONS: - lines.append(style.section("downstream")) - lines.extend( - render_dependency_tree( - focus=focus, - deps=downstream, - seen={focus}, - format_node=lambda key: _format_node(node=node_by_key[key], style=style), - sort_key=lambda key: (str(key.resource_type), key.name), - branch_style=style.muted, - already_shown=lambda: style.muted(" (already shown)"), - ) - ) - return "\n".join(lines) - - -def format_column_lineage_tree( - *, - trace: ColumnLineageTrace, - use_color: bool = True, -) -> str: - """Format column lineage for humans without graph implementation terms.""" - - style: CliStyle = CliStyle(use_color=use_color) - title: str = style.title("Column trace") - direction: str = style.muted(trace.direction) - lines: list[str] = [ - f"{title} {_format_column(column=trace.target, style=style)} {direction}", - "", - ] - if not trace.trace: - lines.append(style.muted(" No column dependencies found")) - return "\n".join(lines) - is_downstream: bool = trace.direction == DOWNSTREAM_DIRECTION - deps: dict[str, list[ColumnLineageEdge]] = {} - for edge in trace.trace[:_HUMAN_COLUMN_TRACE_LIMIT]: - key_column: QualifiedLineageColumn = edge.source if is_downstream else edge.target - deps.setdefault(_column_id(key_column), []).append(edge) - lines.extend( - render_column_trace_tree( - target=trace.target, - deps=deps, - total=len(trace.trace), - limit=_HUMAN_COLUMN_TRACE_LIMIT, - column_id=_column_id, - related_column=lambda edge: edge.target if is_downstream else edge.source, - format_related=lambda edge: _format_related_column( - edge=edge, is_downstream=is_downstream, style=style - ), - branch_style=style.muted, - already_shown=lambda: style.muted(" (already shown)"), - note_style=style.muted, - ) - ) - return "\n".join(lines) - - -def _format_related_column(*, edge: ColumnLineageEdge, is_downstream: bool, style: CliStyle) -> str: - related_column: QualifiedLineageColumn = edge.target if is_downstream else edge.source - return ( - f"{_format_column(column=related_column, style=style)} " - f"{style.muted(f'({_human_transform_label(edge)})')}" - ) - - -def _format_graph_summary(*, graph: LineageGraph, use_color: bool) -> str: - style: CliStyle = CliStyle(use_color=use_color) - title: str = style.title("Lineage graph") - counts: str = style.muted(f"({len(graph.nodes)} nodes, {len(graph.edges)} edges)") - lines: list[str] = [f"{title} {counts}"] - if graph.edges: - lines.extend( - f"{style.muted(' - ')}" - f"{_format_key(key=parent, style=style)} " - f"{style.muted('->')} " - f"{_format_key(key=child, style=style)}" - for parent, child in graph.edges - ) - else: - lines.extend(f" {_format_node(node=node, style=style)}" for node in graph.nodes) - return "\n".join(lines) - - -def _serialize_node(node: LineageNode) -> dict[str, object]: - payload: dict[str, object] = { - "id": _node_id(node.key), - "name": node.key.name, - "resource_type": str(node.key.resource_type), - } - if node.relative_path is not None: - payload["relative_path"] = node.relative_path - if node.qualified_name is not None: - payload["qualified_name"] = node.qualified_name - return payload - - -def _render_resource_type(column: QualifiedLineageColumn) -> str: - return CompiledResourceType(column.resource_type).value - - -def _format_transform(*, edge: ColumnLineageEdge, style: CliStyle) -> str: - return style.muted(_human_transform_label(edge)) - - -def _human_transform_label(edge: ColumnLineageEdge) -> str: - if str(edge.transform_kind) == ColumnTransformKind.STAR: - return "from SELECT *" - return str(edge.transform_kind) - - -def _format_node(*, node: LineageNode, style: CliStyle) -> str: - parts: list[str] = [ - style.muted(str(node.key.resource_type)), - style.section(node.key.name), - ] - if node.relative_path is not None: - parts.append(style.muted(node.relative_path)) - return " ".join(parts) - - -def _format_key(*, key: CompiledObjectKey, style: CliStyle) -> str: - return f"{style.muted(str(key.resource_type))}:{style.section(key.name)}" - - -def _format_column(*, column: QualifiedLineageColumn, style: CliStyle) -> str: - return style.section(_column_id(column)) - - -def _column_id(column: QualifiedLineageColumn) -> str: - return f"{column.resource_name}.{column.column_name}" - - -def _node_id(key: CompiledObjectKey) -> str: - return f"{key.resource_type}:{key.name}" diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/selection.py b/src/sqlbuild/cli/commands/_helpers/lineage/selection.py deleted file mode 100644 index 67917f981b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/lineage/selection.py +++ /dev/null @@ -1,558 +0,0 @@ -"""Lineage graph selection helpers.""" - -from __future__ import annotations - -from collections.abc import Iterable -from dataclasses import dataclass - -import sqlbuild._native as _native -from sqlbuild._native import ( - ProjectColumnLineage, - path_nodes, - transitive_closure, - transitive_closure_many, -) -from sqlbuild.cli.commands.constants import ( - BOTH_DIRECTIONS, - COLUMN_TARGET_KIND, - COLUMN_TARGET_SEPARATOR, - DOWNSTREAM_DIRECTION, - RICH_LINEAGE_STATUS_MODEL_THRESHOLD, - SELECTOR_INTERSECTION_MARKER, - UNLIMITED_DEPTH_VALUE, - UPSTREAM_DIRECTION, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - ColumnLineageTrace, - LineageGraph, - LineageNode, - LineageSelectionAnchors, - LineageTarget, - RelationLineageIndex, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.lineage.models import ( - ColumnLineageEdge, - QualifiedLineageColumn, -) -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors -from sqlbuild.compiler.planner.main.selection.selector_match import match_project_selector -from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector -from sqlbuild.compiler.planner.main.selection.selector_tokens import ( - resolve_project_selector_tokens, -) -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector -from sqlbuild.compiler.planner.types import SelectorKind -from sqlbuild.presentation.main.maybe_status import maybe_status - - -@dataclass(frozen=True) -class _ColumnLineageCandidateSelection: - model_names: frozenset[str] - truncated: bool - - -_KIND_SEPARATOR: str = ":" -_GRAPH_OPERATORS: tuple[str, ...] = ("+", "@", "*") -_TARGET_KINDS: frozenset[str] = frozenset( - { - str(CompiledResourceType.MODEL), - str(CompiledResourceType.SOURCE), - str(CompiledResourceType.SEED), - str(CompiledResourceType.UDF), - str(CompiledResourceType.TABLE_FN), - COLUMN_TARGET_KIND, - } -) - - -def parse_depth(raw_depth: str) -> int | None: - """Parse a lineage depth value, returning None for unlimited traversal.""" - - if raw_depth == UNLIMITED_DEPTH_VALUE: - return None - try: - depth: int = int(raw_depth) - except ValueError: - raise CliUserError("--depth must be a non-negative integer or 'all'", code="C304") from None - if depth < 0: - raise CliUserError("--depth must be a non-negative integer or 'all'", code="C304") - return depth - - -def normalize_lineage_target(*, graph: ProjectGraph | RelationLineageIndex, target: str) -> str: - """Strip a printed kind prefix such as `model:` after checking it matches the resource.""" - - resolved: LineageTarget = resolve_lineage_target(all_keys=graph.all_keys, target=target) - if resolved.kind is None: - return target - if not resolved.kind_matches: - raise _unknown_target_error(target) - return resolved.name - - -def resolve_lineage_target(*, all_keys: dict[str, CompiledObjectKey], target: str) -> LineageTarget: - """Resolve `[kind:]` or `[kind:].` against project resource names.""" - - kind, separator, remainder = target.partition(_KIND_SEPARATOR) - written_kind: str | None = kind if separator and kind in _TARGET_KINDS else None - name: str = remainder if written_kind is not None else target - resource_name, column_separator, column_name = name.partition(COLUMN_TARGET_SEPARATOR) - key: CompiledObjectKey | None = all_keys.get(resource_name) - column: str | None = column_name if column_separator else None - kind_matches: bool - if written_kind is None: - kind_matches = True - elif written_kind == COLUMN_TARGET_KIND: - kind_matches = ( - key is not None and key.resource_type == CompiledResourceType.MODEL and bool(column) - ) - else: - kind_matches = key is not None and str(key.resource_type) == written_kind - return LineageTarget( - kind=written_kind, - name=name, - key=key, - column_name=column, - kind_matches=kind_matches, - ) - - -def select_target_lineage( - *, - graph: ProjectGraph | RelationLineageIndex, - targets: tuple[str, ...], - direction: str, - depth: int | None, -) -> LineageGraph: - """Select lineage around one or more positional targets.""" - - keys: tuple[CompiledObjectKey, ...] = tuple( - _lookup_name(name=target, all_keys=graph.all_keys) for target in targets - ) - selected: set[CompiledObjectKey] = set(keys) - selected.update(_expand(keys=keys, graph=graph, direction=direction, depth=depth)) - return build_lineage_graph( - graph=graph, - selected_keys=frozenset(selected), - focus_keys=tuple(sorted(set(keys), key=_sort_key)), - direction=direction, - ) - - -def _expand( - *, - keys: Iterable[CompiledObjectKey], - graph: ProjectGraph | RelationLineageIndex, - direction: str, - depth: int | None, -) -> frozenset[CompiledObjectKey]: - anchors: tuple[CompiledObjectKey, ...] = tuple(keys) - expanded: set[CompiledObjectKey] = set() - if direction in {UPSTREAM_DIRECTION, BOTH_DIRECTIONS}: - expanded.update( - transitive_closure_many( - starts=anchors, edges=graph.upstream_deps, include_starts=False, max_depth=depth - ) - ) - if direction in {DOWNSTREAM_DIRECTION, BOTH_DIRECTIONS}: - expanded.update( - transitive_closure_many( - starts=anchors, edges=graph.downstream_deps, include_starts=False, max_depth=depth - ) - ) - return frozenset(expanded) - - -def _unknown_target_error(target: str) -> CliUserError: - uses_graph_operator: bool = any(operator in target for operator in _GRAPH_OPERATORS) - return CliUserError( - f"unknown lineage target '{target}'", - code="C305", - help=( - "graph operators are not supported in lineage targets; " - "use --direction upstream|downstream|both and --depth N" - if uses_graph_operator - else "use a model, source or seed name, optionally prefixed like model:orders" - ), - ) - - -def select_column_target_lineage( - *, - graph: ProjectGraph | RelationLineageIndex, - target: str, - direction: str, - depth: int | None, - mode: ColumnLineageMode = ColumnLineageMode.RICH, -) -> ColumnLineageTrace | None: - """Select column-level lineage when target uses model.column syntax.""" - - if COLUMN_TARGET_SEPARATOR not in target: - return None - if not isinstance(graph, ProjectGraph): - raise CliUserError( - "column lineage requires a compiled project graph", - code="C307", - help="enable SQL analysis or install SQLBuild with Polyglot support", - ) - resource_name: str - column_name: str - resource_name, column_name = target.rsplit(COLUMN_TARGET_SEPARATOR, 1) - key: CompiledObjectKey | None = graph.all_keys.get(resource_name) - if key is None or key.resource_type != CompiledResourceType.MODEL: - return None - if direction == BOTH_DIRECTIONS: - raise CliUserError( - "column lineage supports --direction upstream or downstream, not both", - code="C306", - ) - - candidate_selection: _ColumnLineageCandidateSelection = _column_lineage_candidate_selection( - graph=graph, - key=key, - direction=direction, - depth=depth, - ) - with maybe_status( - message=f"Analyzing column lineage for {len(candidate_selection.model_names)} models...", - enabled=( - mode == ColumnLineageMode.RICH - and len(candidate_selection.model_names) >= RICH_LINEAGE_STATUS_MODEL_THRESHOLD - ), - ): - column_lineage: ProjectColumnLineage | None = _native.build_project_column_lineage( - graph.project, None, mode, candidate_selection.model_names - ) - if column_lineage is None: - raise CliUserError( - "column lineage requires SQL analysis to be enabled and available", - code="C307", - help="enable SQL analysis or install SQLBuild with Polyglot support", - ) - - target_column: QualifiedLineageColumn = QualifiedLineageColumn( - resource_type=key.resource_type, - resource_name=key.name, - column_name=column_name, - ) - trace: tuple[ColumnLineageEdge, ...] = _trace_column_with_depth( - column_lineage=column_lineage, - resource_name=key.name, - column_name=column_name, - direction=direction, - max_depth=depth, - ) - return ColumnLineageTrace( - target=target_column, - trace=trace, - direction=direction, - mode=mode, - max_depth=depth, - analyzed_model_count=len(candidate_selection.model_names), - truncated=candidate_selection.truncated, - ) - - -def _column_lineage_candidate_selection( - *, - graph: ProjectGraph, - key: CompiledObjectKey, - direction: str, - depth: int | None, -) -> _ColumnLineageCandidateSelection: - selected: set[CompiledObjectKey] = {key} - deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - graph.downstream_deps if direction == DOWNSTREAM_DIRECTION else graph.upstream_deps - ) - selected.update(transitive_closure(start=key, edges=deps, max_depth=depth)) - model_names: frozenset[str] = frozenset( - selected_key.name - for selected_key in selected - if selected_key.resource_type == CompiledResourceType.MODEL - ) - if depth is None: - return _ColumnLineageCandidateSelection(model_names=model_names, truncated=False) - extended: set[CompiledObjectKey] = {key} - extended.update(transitive_closure(start=key, edges=deps, max_depth=depth + 1)) - return _ColumnLineageCandidateSelection( - model_names=model_names, - truncated=extended != selected, - ) - - -def _trace_column_with_depth( - *, - column_lineage: ProjectColumnLineage, - resource_name: str, - column_name: str, - direction: str, - max_depth: int | None, -) -> tuple[ColumnLineageEdge, ...]: - if max_depth == 0: - return () - result: list[ColumnLineageEdge] = [] - stack: list[tuple[str, str, int]] = [(resource_name, column_name, 0)] - visited: set[tuple[str, str]] = set() - while stack: - current_resource: str - current_column: str - current_depth: int - current_resource, current_column, current_depth = stack.pop() - if (current_resource, current_column) in visited: - continue - visited.add((current_resource, current_column)) - if max_depth is not None and current_depth >= max_depth: - continue - if direction == DOWNSTREAM_DIRECTION: - edges: tuple[ColumnLineageEdge, ...] = column_lineage.column_consumers( - resource_name=current_resource, - column_name=current_column, - ) - for edge in edges: - result.append(edge) - stack.append( - (edge.target.resource_name, edge.target.column_name, current_depth + 1) - ) - else: - for edge in column_lineage.edges_targeting(current_resource): - if edge.target.column_name != current_column: - continue - result.append(edge) - stack.append( - (edge.source.resource_name, edge.source.column_name, current_depth + 1) - ) - return tuple(result) - - -def select_selector_lineage( - *, - graph: ProjectGraph | RelationLineageIndex, - select: tuple[str, ...], - exclude: tuple[str, ...], - depth: int | None, - direction: str | None = None, -) -> LineageGraph: - """Select lineage using the shared project selector syntax, expanding by direction if given.""" - - selected_keys: frozenset[CompiledObjectKey] = resolve_project_selectors( - select=select, - exclude=exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, - ) - if direction is not None: - excluded_keys: frozenset[CompiledObjectKey] = resolve_project_selector_tokens( - selectors=exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, - ) - return build_lineage_graph( - graph=graph, - selected_keys=( - selected_keys - | _expand(keys=selected_keys, graph=graph, direction=direction, depth=depth) - ) - - excluded_keys, - focus_keys=tuple(sorted(selected_keys - excluded_keys, key=_sort_key)), - direction=direction, - ) - anchors: LineageSelectionAnchors = _resolve_selector_anchors( - select=select, - graph=graph, - require_clear_anchors=depth is not None, - ) - if depth is not None: - selected_keys = _trim_selected_keys( - selected_keys=selected_keys, - anchors=anchors, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - max_depth=depth, - ) - focus_keys: tuple[CompiledObjectKey, ...] = tuple( - sorted(anchors.upstream | anchors.downstream, key=_sort_key) - ) - return build_lineage_graph( - graph=graph, - selected_keys=selected_keys, - focus_keys=focus_keys, - direction=None, - ) - - -def build_lineage_graph( - *, - graph: ProjectGraph | RelationLineageIndex, - selected_keys: frozenset[CompiledObjectKey], - focus_keys: tuple[CompiledObjectKey, ...] = (), - direction: str | None = None, -) -> LineageGraph: - """Build display nodes and selected edges.""" - - nodes: tuple[LineageNode, ...] = tuple( - _build_node(graph=graph, key=key) for key in sorted(selected_keys, key=_sort_key) - ) - selected_edges: list[tuple[CompiledObjectKey, CompiledObjectKey]] = [] - for downstream_key in sorted(selected_keys, key=_sort_key): - for upstream_key in graph.upstream_deps.get(downstream_key, ()): - if upstream_key in selected_keys: - selected_edges.append((upstream_key, downstream_key)) - return LineageGraph( - nodes=nodes, - edges=tuple(selected_edges), - focus_keys=focus_keys, - direction=direction, - ) - - -def _resolve_selector_anchors( - *, - select: tuple[str, ...], - graph: ProjectGraph | RelationLineageIndex, - require_clear_anchors: bool, -) -> LineageSelectionAnchors: - """Find depth anchors for selectors already validated by the shared resolver.""" - - upstream_anchors: set[CompiledObjectKey] = set() - downstream_anchors: set[CompiledObjectKey] = set() - retained: set[CompiledObjectKey] = set() - - for raw_select in select: - for token in raw_select.split(): - if SELECTOR_INTERSECTION_MARKER in token and require_clear_anchors: - raise CliUserError( - "--depth cannot be combined with comma-intersection selectors", - code="C308", - ) - for part in token.split(SELECTOR_INTERSECTION_MARKER): - parsed: ParsedSelector | PathSelector = parse_project_selector(part) - if isinstance(parsed, PathSelector): - start_key: CompiledObjectKey = graph.all_keys[parsed.start_name] - end_key: CompiledObjectKey = graph.all_keys[parsed.end_name] - retained.update( - path_nodes(start=start_key, end=end_key, downstream=graph.downstream_deps) - or () - ) - upstream_anchors.add(start_key) - downstream_anchors.add(end_key) - continue - if parsed.kind in {SelectorKind.TAG, SelectorKind.PATH} and require_clear_anchors: - raise CliUserError( - "--depth requires name, source, seed, or path-between selectors", - code="C309", - ) - matched: frozenset[CompiledObjectKey] = match_project_selector( - parsed=parsed, - all_keys=graph.all_keys, - tag_index=graph.tag_index, - path_index=graph.path_index, - ) - if parsed.kind in {SelectorKind.TAG, SelectorKind.PATH}: - upstream_anchors.update(matched) - downstream_anchors.update(matched) - continue - if parsed.upstream: - upstream_anchors.update(matched) - if parsed.downstream: - downstream_anchors.update(matched) - if not parsed.upstream and not parsed.downstream: - retained.update(matched) - return LineageSelectionAnchors( - upstream=frozenset(upstream_anchors), - downstream=frozenset(downstream_anchors), - retained=frozenset(retained), - ) - - -def _trim_selected_keys( - *, - selected_keys: frozenset[CompiledObjectKey], - anchors: LineageSelectionAnchors, - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - max_depth: int, -) -> frozenset[CompiledObjectKey]: - retained: set[CompiledObjectKey] = set(anchors.retained) - retained.update(anchors.upstream) - retained.update(anchors.downstream) - retained.update( - transitive_closure_many( - starts=anchors.upstream, edges=upstream_deps, include_starts=False, max_depth=max_depth - ) - ) - retained.update( - transitive_closure_many( - starts=anchors.downstream, - edges=downstream_deps, - include_starts=False, - max_depth=max_depth, - ) - ) - return frozenset(selected_keys & retained) - - -def _lookup_name(*, name: str, all_keys: dict[str, CompiledObjectKey]) -> CompiledObjectKey: - key: CompiledObjectKey | None = all_keys.get(name) - if key is None: - raise _unknown_target_error(name) - return key - - -def _build_node( - *, graph: ProjectGraph | RelationLineageIndex, key: CompiledObjectKey -) -> LineageNode: - if isinstance(graph, RelationLineageIndex): - return graph.nodes.get(key, LineageNode(key=key)) - project: CompiledProjectView = graph.project - for model in project.models: - if model.key == key: - return LineageNode( - key=key, - relative_path=str(model.relative_path), - qualified_name=model.destination.qualified_name, - ) - for seed in project.seeds: - if seed.key == key: - return LineageNode( - key=key, - relative_path=str(seed.seed_file.relative_path), - qualified_name=seed.destination.qualified_name, - ) - for source in project.sources: - if source.key == key: - return LineageNode( - key=key, - relative_path=str(source.source_file.relative_path), - qualified_name=_source_relation_name(source.source_entry), - ) - return LineageNode(key=key) - - -def _source_relation_name(source: object) -> str | None: - database: str | None = getattr(source, "database", None) - schema: str | None = getattr(source, "schema", None) - table: str | None = getattr(source, "table", None) - if table is None: - return None - if database is not None and schema is not None: - return f"{database}.{schema}.{table}" - if schema is not None: - return f"{schema}.{table}" - return table - - -def _sort_key(key: CompiledObjectKey) -> tuple[str, str]: - return (str(key.resource_type), key.name) diff --git a/src/sqlbuild/cli/commands/_helpers/load/execution.py b/src/sqlbuild/cli/commands/_helpers/load/execution.py index ef463fc56d..c3617fe427 100644 --- a/src/sqlbuild/cli/commands/_helpers/load/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/load/execution.py @@ -4,6 +4,7 @@ import time +from sqlbuild._native import ConnectionProgressReporter, ExecutionEventWriter from sqlbuild.cli.commands._helpers.input.parsing import ( parse_cursor_integer, parse_cursor_timestamp, @@ -15,8 +16,6 @@ LoadInvocation, LoadRunOutcome, ) -from sqlbuild.cli.output.classes.execution_event_writer import ExecutionEventWriter -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter from sqlbuild.executor.load.main.run import run_load_pipeline from sqlbuild.executor.load.models import ( LoadCallbacks, diff --git a/src/sqlbuild/cli/commands/_helpers/load/outputs.py b/src/sqlbuild/cli/commands/_helpers/load/outputs.py index 49e6157798..1f1f2e4247 100644 --- a/src/sqlbuild/cli/commands/_helpers/load/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/load/outputs.py @@ -2,7 +2,13 @@ from __future__ import annotations -from sqlbuild._native import format_load_execution_json, write_execution_json_output +from sqlbuild._native import ( + format_completion_line, + format_load_execution_json, + format_summary_footer, + write_execution_header, + write_execution_json_output, +) from sqlbuild.cli.commands.classes.load_progress_reporter import format_load_footer from sqlbuild.cli.commands.models import ( LoadCommandRequest, @@ -10,10 +16,7 @@ LoadInvocation, LoadRunOutcome, ) -from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.completion_line import format_completion_line -from sqlbuild.presentation.main.summary_footer import format_summary_footer from sqlbuild.presentation.main.surface_header import format_surface_header from sqlbuild.presentation.main.tree_connector import tree_connector from sqlbuild.presentation.types import CompletionState diff --git a/src/sqlbuild/cli/commands/_helpers/load/planning.py b/src/sqlbuild/cli/commands/_helpers/load/planning.py index 254c7aa85b..98beea16dc 100644 --- a/src/sqlbuild/cli/commands/_helpers/load/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/load/planning.py @@ -2,13 +2,13 @@ from __future__ import annotations -from sqlbuild._native import render_source_relation +from sqlbuild._native import ( + render_source_relation, + resolve_project_connection_config, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.load.references import validate_reference_source_targets from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, -) from sqlbuild.cli.commands.models import ( LoadCommandRequest, LoadExecutionPreparation, diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py index d331b6c0aa..5a1a66cd8e 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py @@ -6,11 +6,10 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, validate_preview_target from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) -from sqlbuild.cli.commands._helpers.runtime.preview_target import validate_preview_target from sqlbuild.cli.commands.models import ( AdapterConnectionContext, PlanCommandRequest, diff --git a/src/sqlbuild/cli/commands/_helpers/playground/__init__.py b/src/sqlbuild/cli/commands/_helpers/playground/__init__.py deleted file mode 100644 index 1c92622a76..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/playground/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Playground command helpers.""" diff --git a/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py b/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py deleted file mode 100644 index e8d1d6f59c..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Playground completion output rendering phase.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.constants import ( - PLAYGROUND_ORCHESTRATED_PROJECT_DIR, - PLAYGROUND_ORCHESTRATED_TEMPLATES, -) -from sqlbuild.cli.commands.models import ( - PlaygroundCommandRequest, - PlaygroundTarget, -) -from sqlbuild.cli.commands.types import PlaygroundTemplate -from sqlbuild.presentation.classes.cli_document import CliDocument -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.supports_color import supports_color - - -def render_playground_completion_text( - *, - request: PlaygroundCommandRequest, - target: PlaygroundTarget, -) -> str: - """Render the playground creation summary and suggested commands.""" - - display_path: str = str(request.target_path) - style: CliStyle = CliStyle(use_color=supports_color()) - doc: CliDocument = CliDocument(style) - doc.header(text="SQLBuild playground created") - doc.blank() - doc.field(label="Project", value=display_path) - doc.field(label="Adapter", value="DuckDB") - doc.field(label="Example", value=_example_name(template=target.template)) - doc.blank() - doc.title_section("Try") - doc.commands( - commands=_suggested_commands(template=target.template, display_path=display_path), - style_command=False, - ) - return doc.render() - - -def _suggested_commands(*, template: PlaygroundTemplate, display_path: str) -> tuple[str, ...]: - orchestrated: bool = template in PLAYGROUND_ORCHESTRATED_TEMPLATES - project_path: str = ( - f"{display_path}/{PLAYGROUND_ORCHESTRATED_PROJECT_DIR}" if orchestrated else display_path - ) - commands: list[str] = [f"cd {project_path}"] - if template == PlaygroundTemplate.PYTHON_NODES: - commands.extend( - [ - "sqb plan --select +fact_orders --select +orders_export", - "sqb build --select +fact_orders --select +orders_export", - "sqb check --select check_orders_export", - ] - ) - else: - commands.extend(["sqb compile", "sqb build", "sqb test", "sqb audit"]) - if orchestrated: - commands.append("cd ..") - if template == PlaygroundTemplate.DAGSTER: - commands.append("DAGSTER_IS_DEV_CLI=1 dagster dev -f definitions.py") - if template == PlaygroundTemplate.RIVERS: - commands.append("rivers dev definitions") - return tuple(commands) - - -def _example_name(template: PlaygroundTemplate) -> str: - labels: dict[PlaygroundTemplate, str] = { - PlaygroundTemplate.DAGSTER: "waffle shop + Dagster", - PlaygroundTemplate.RIVERS: "waffle shop + Rivers", - PlaygroundTemplate.LOADER_WAFFLE_SHOP: "loader-focused waffle shop", - PlaygroundTemplate.PYTHON_NODES: "Python nodes demo", - } - return labels.get(template, "waffle shop") diff --git a/src/sqlbuild/cli/commands/_helpers/playground/copy.py b/src/sqlbuild/cli/commands/_helpers/playground/copy.py deleted file mode 100644 index cc0fd8c7c4..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/playground/copy.py +++ /dev/null @@ -1,446 +0,0 @@ -"""Copy packaged playground templates to user projects.""" - -from __future__ import annotations - -from importlib.resources import files -from importlib.resources.abc import Traversable -from pathlib import Path - -from sqlbuild.cli.commands.constants import ( - PLAYGROUND_ORCHESTRATED_PROJECT_DIR, - PLAYGROUND_TEMPLATE_VALUES, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.types import PlaygroundTemplate -from sqlbuild.compiler.discovery.constants import PYTHON_NODE_ROOT - -_TEMPLATE_PACKAGE: str = "sqlbuild.cli.commands._helpers.playground" -_WAFFLE_SHOP_TEMPLATE: str = "templates/waffle_shop" -_LOADER_WAFFLE_SHOP_TEMPLATE: str = "templates/loader_waffle_shop" -_ORCHESTRATOR_DEFINITIONS_FILE: str = "definitions.py" -_ORCHESTRATOR_README_FILE: str = "README.md" - -_PYTHON_NODES_PROJECT_TOML: str = """name = "python_nodes_demo" -adapter = "duckdb" -default_target = "dev" - -[connections.developer] -database = "python_nodes_demo.duckdb" - -[settings] -default_audit_severity = "warn" - -[defaults] -materialized = "table" - -[targets.dev] -connection = "developer" -schema = "main" -""" - -_PYTHON_NODES_README: str = """# SQLBuild Python Nodes Playground - -This project demonstrates first-class Python nodes in a small local DuckDB project. - -Try: - -```bash -sqb plan --select +fact_orders --select +orders_export -sqb build --select +fact_orders --select +orders_export -sqb check --select check_orders_export -``` - -The project includes a task feeding a loader, a model feeding Python assets through -`ctx.relation(model(...))`, soft skip fan-in, `materialized=False`, and a Python check. -""" - -_PYTHON_NODES_SOURCES_YML: str = """sources: - - name: raw_orders - description: Raw orders loaded by the raw_orders Python loader - managed: true - write_strategy: table - columns: - - name: order_id - type: INTEGER - - name: customer_id - type: INTEGER - - name: amount_cents - type: INTEGER - - name: load_batch - type: INTEGER -""" - -_PYTHON_NODES_FACT_ORDERS_SQL: str = """MODEL ( - description "One row per order with its customer and amount", - materialized table, - columns ( - order_id (nullable false, audits [not_null, unique]), - customer_id (nullable false, audits [not_null]), - amount_cents (nullable false, audits [not_null]), - ), -); - -SELECT - order_id, - customer_id, - amount_cents -FROM __source("raw_orders") -""" - -_PYTHON_NODES_TASKS_PY: str = '''"""Task examples for the Python nodes playground.""" - -from sqlbuild.tasks import SkipMode, task - - -@task(tags=["python", "ingress"], group="ingress") -def prepare_raw_orders(ctx): - """Prepare a tiny in-memory order extract before the loader runs.""" - - rows = [ - {"order_id": 1, "customer_id": 10, "amount_cents": 1200, "load_batch": 1}, - {"order_id": 2, "customer_id": 20, "amount_cents": 800, "load_batch": 1}, - ] - return ctx.result(payload={"rows": rows}, metadata={"row_count": len(rows)}) - - -@task(tags=["python", "optional"], group="exports") -def optional_partner_feed(ctx): - """Soft-skip an optional upstream without blocking sibling fan-in.""" - - return ctx.skip(reason="partner feed is not configured", mode=SkipMode.SOFT) - - -@task(tags=["python", "optional"], group="exports") -def export_window(ctx): - """Successful sibling that lets downstream fan-in continue.""" - - return ctx.result(payload={"window": "daily"}) -''' - -_PYTHON_NODES_LOADERS_PY: str = '''"""Loader examples for the Python nodes playground.""" - -from sqlbuild.loaders import loader -from python.tasks.orders import prepare_raw_orders - - -@loader(depends_on=(prepare_raw_orders,)) -def raw_orders(ctx): - """Load source rows after the preparation task has completed.""" - - # Loader contexts intentionally keep the SQL ingress API narrow; the dependency makes - # the task part of the same DAG even though this example returns static rows. - return [ - {"order_id": 1, "customer_id": 10, "amount_cents": 1200, "load_batch": 1}, - {"order_id": 2, "customer_id": 20, "amount_cents": 800, "load_batch": 1}, - ] -''' - -_PYTHON_NODES_ASSETS_PY: str = '''"""Asset examples for the Python nodes playground.""" - -from sqlbuild.assets import asset -from sqlbuild.refs import model - -from python.tasks.orders import export_window, optional_partner_feed - - -@asset( - depends_on=(model("fact_orders"), optional_partner_feed, export_window), - tags=["python", "external"], - group="exports", - columns=[{"name": "order_count", "type": "INTEGER"}], - column_lineage={"order_count": [{"node": "fact_orders", "column": "order_id"}]}, -) -def orders_export(ctx): - """Read a SQL model relation and describe an external export artifact.""" - - relation = ctx.relation(model("fact_orders")) - rows = ctx.query(f"SELECT COUNT(*) AS order_count FROM {relation}").fetchall() - order_count = int(rows[0][0]) - return ctx.result( - payload={"order_count": order_count}, - metadata={"target_uri": "s3://example-bucket/orders.json", "window": "daily"}, - materialized=False, - ) -''' - -_PYTHON_NODES_CHECKS_PY: str = '''"""Check examples for the Python nodes playground.""" - -from sqlbuild.checks import check - -from python.assets.orders_export import orders_export - - -@check(depends_on=orders_export, tags=["quality"], group="exports") -def check_orders_export(ctx): - """Validate the Python asset's same-run metadata and payload.""" - - result = ctx.result_of(node_function=orders_export) - payload = result.payload - metadata = result.metadata - if payload["order_count"] <= 0: - return ctx.fail(message="orders export is empty") - if "target_uri" not in metadata: - return ctx.warn(message="orders export target URI is missing") - return ctx.pass_( - message="orders export is ready", - metadata={"order_count": payload["order_count"]}, - ) -''' - - -_DAGSTER_DEFINITIONS: str = '''"""Dagster definitions for the SQLBuild waffle shop playground.""" - -from pathlib import Path - -import dagster as dg - -from sqlbuild.integrations.dagster import ( - SqlBuildCliResource, - SqlBuildProject, - sqlbuild_assets, - sqlbuild_scenario_checks, -) - -PROJECT_DIR = Path(__file__).resolve().parent / "waffle_shop" -SQLBUILD_PROJECT = SqlBuildProject(project_dir=PROJECT_DIR) - -SQLBUILD_PROJECT.prepare_if_dev() - - -@sqlbuild_assets( - project=SQLBUILD_PROJECT, - required_resource_keys={"sqb"}, - include_scenario_checks=False, -) -def waffle_shop_assets(context: dg.AssetExecutionContext): - yield from context.resources.sqb.cli(["build"], context=context).stream() - - -@sqlbuild_scenario_checks( - project=SQLBUILD_PROJECT, - required_resource_keys={"sqb"}, -) -def waffle_shop_scenarios(context: dg.AssetCheckExecutionContext): - yield from context.resources.sqb.cli(["scenario", "test"], context=context).stream() - - -defs = dg.Definitions( - assets=[waffle_shop_assets], - asset_checks=[waffle_shop_scenarios], - resources={"sqb": SqlBuildCliResource(project_dir=SQLBUILD_PROJECT)}, -) -''' - -_DAGSTER_README: str = """# SQLBuild + Dagster playground - -This directory is a Dagster code location. `definitions.py` orchestrates the SQLBuild waffle -shop project in `waffle_shop/`, which stays a plain SQLBuild project with no Dagster code inside. - -## Run Dagster - -Install SQLBuild with the Dagster extra, then start Dagster from this directory: - -```bash -uv add "sqlbuild[dagster]" -DAGSTER_IS_DEV_CLI=1 uv run dagster dev -f definitions.py -``` - -Open the Dagster UI, materialize the `waffle_shop_assets` asset group, then run the scenario checks. - -The definitions use `SqlBuildProject.prepare_if_dev()` to generate -`waffle_shop/target/sqlbuild_dag.json` when Dagster starts in dev mode. -""" - -_RIVERS_DEFINITIONS: str = '''"""Rivers definitions for the SQLBuild waffle shop playground.""" - -from __future__ import annotations - -import subprocess -from collections.abc import Iterator -from pathlib import Path -from typing import Any - -import rivers as rs - -from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets - -PROJECT_DIR = Path(__file__).resolve().parent / "waffle_shop" -SQLBUILD_PROJECT = SqlBuildProject(project_dir=PROJECT_DIR) - -if __name__ == "__main__": - SQLBUILD_PROJECT.prepare() -else: - SQLBUILD_PROJECT.prepare_if_dev() - - -@sqlbuild_assets(project=SQLBUILD_PROJECT) -def waffle_shop_assets(context: Any) -> Iterator[Any]: - completed = subprocess.run( - ["sqb", "build"], - cwd=PROJECT_DIR, - capture_output=True, - check=False, - text=True, - ) - if completed.returncode != 0: - raise RuntimeError(completed.stderr or completed.stdout) - for output_name in context.output_selection: - yield rs.Materialization(output_name=output_name) - - -repo = rs.CodeRepository( - assets=[waffle_shop_assets], - jobs=[ - rs.Job( - name="waffle_shop", - assets=[waffle_shop_assets], - executor=rs.Executor.in_process(), - ), - ], - default_executor=rs.Executor.in_process(), -) - - -if __name__ == "__main__": - result = repo.get_job("waffle_shop").execute() - raise SystemExit(0 if result.success else 1) -''' - -_RIVERS_README: str = """# SQLBuild + Rivers playground - -This directory is a Rivers code repository. `definitions.py` orchestrates the SQLBuild waffle -shop project in `waffle_shop/`, which stays a plain SQLBuild project with no Rivers code inside. - -## Run Rivers - -Install SQLBuild with the Rivers extra, then start Rivers from this directory: - -```bash -uv add "sqlbuild[rivers]" -uv run rivers dev definitions -``` - -You can also run a local materialization directly: - -```bash -uv run python definitions.py -``` - -The repository defines a `waffle_shop` job for UI-triggered materialization. - -The definitions use `SqlBuildProject.prepare_if_dev()` to generate -`waffle_shop/target/sqlbuild_dag.json` when `rivers dev` starts with `RIVERS_DEPLOYMENT=dev`. -""" - - -def create_playground_project( - *, target_dir: Path, template: str = PlaygroundTemplate.WAFFLE_SHOP.value -) -> None: - """Create a DuckDB-backed waffle shop playground project.""" - - if target_dir.exists(): - raise CliUserError( - f"playground target already exists: {target_dir}", - code="C701", - help="choose a new directory or remove the existing one", - ) - if template not in PLAYGROUND_TEMPLATE_VALUES: - raise CliUserError( - f"unknown playground template: {template}", - code="C703", - help=f"choose one of: {', '.join(PLAYGROUND_TEMPLATE_VALUES)}", - ) - resolved_template: PlaygroundTemplate = PlaygroundTemplate(template) - - if resolved_template == PlaygroundTemplate.PYTHON_NODES: - _write_python_nodes_template_files(target_dir=target_dir) - return - - template_path: str = ( - _LOADER_WAFFLE_SHOP_TEMPLATE - if resolved_template == PlaygroundTemplate.LOADER_WAFFLE_SHOP - else _WAFFLE_SHOP_TEMPLATE - ) - template_root: Traversable = files(_TEMPLATE_PACKAGE).joinpath(template_path) - if not template_root.is_dir(): - raise CliUserError( - "packaged playground template is missing", - code="C702", - help="reinstall SQLBuild or report a packaging issue", - ) - if resolved_template == PlaygroundTemplate.DAGSTER: - _write_orchestrator_template_files( - template_root=template_root, - target_dir=target_dir, - definitions=_DAGSTER_DEFINITIONS, - readme=_DAGSTER_README, - ) - return - if resolved_template == PlaygroundTemplate.RIVERS: - _write_orchestrator_template_files( - template_root=template_root, - target_dir=target_dir, - definitions=_RIVERS_DEFINITIONS, - readme=_RIVERS_README, - ) - return - _copy_tree(source=template_root, target=target_dir) - - -def _copy_tree(*, source: Traversable, target: Path) -> None: - target.mkdir(parents=True, exist_ok=False) - child: Traversable - for child in source.iterdir(): - child_target: Path = target / _generated_name(child.name) - if child.is_dir(): - _copy_tree(source=child, target=child_target) - continue - child_target.write_bytes(child.read_bytes()) - - -def _generated_name(template_name: str) -> str: - if template_name.endswith(".py.txt"): - return template_name.removesuffix(".txt") - return template_name - - -def _write_orchestrator_template_files( - *, template_root: Traversable, target_dir: Path, definitions: str, readme: str -) -> None: - _copy_tree(source=template_root, target=target_dir / PLAYGROUND_ORCHESTRATED_PROJECT_DIR) - (target_dir / _ORCHESTRATOR_DEFINITIONS_FILE).write_text(definitions, encoding="utf-8") - (target_dir / _ORCHESTRATOR_README_FILE).write_text(readme, encoding="utf-8") - - -def _write_python_nodes_template_files(*, target_dir: Path) -> None: - target_dir.mkdir(parents=True, exist_ok=False) - (target_dir / "sqlbuild_project.toml").write_text( - _PYTHON_NODES_PROJECT_TOML, - encoding="utf-8", - ) - (target_dir / "README.md").write_text(_PYTHON_NODES_README, encoding="utf-8") - sources_dir: Path = target_dir / "sources" - sources_dir.mkdir() - (sources_dir / "raw.yml").write_text(_PYTHON_NODES_SOURCES_YML, encoding="utf-8") - models_dir: Path = target_dir / "models" - models_dir.mkdir() - (models_dir / "fact_orders.sql").write_text( - _PYTHON_NODES_FACT_ORDERS_SQL, - encoding="utf-8", - ) - python_files: tuple[tuple[str, str], ...] = ( - ("tasks/orders.py", _PYTHON_NODES_TASKS_PY), - ("loaders/orders.py", _PYTHON_NODES_LOADERS_PY), - ("assets/orders_export.py", _PYTHON_NODES_ASSETS_PY), - ("checks/orders_export.py", _PYTHON_NODES_CHECKS_PY), - ) - relative_path: str - contents: str - for relative_path, contents in python_files: - python_file: Path = target_dir / PYTHON_NODE_ROOT / relative_path - python_file.parent.mkdir(parents=True, exist_ok=True) - python_file.write_text(contents, encoding="utf-8") - for hook_language in ("sql", "python"): - hook_dir: Path = target_dir / "hooks" / hook_language - hook_dir.mkdir(parents=True) - (hook_dir / ".gitkeep").touch() diff --git a/src/sqlbuild/cli/commands/_helpers/playground/scaffold.py b/src/sqlbuild/cli/commands/_helpers/playground/scaffold.py deleted file mode 100644 index 96954cb9ba..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/playground/scaffold.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Playground target resolution and project scaffolding phases.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.commands._helpers.playground.copy import create_playground_project -from sqlbuild.cli.commands._helpers.skills.update import update_sqlbuild_skills -from sqlbuild.cli.commands.constants import ( - PLAYGROUND_ORCHESTRATED_PROJECT_DIR, - PLAYGROUND_ORCHESTRATED_TEMPLATES, -) -from sqlbuild.cli.commands.models import ( - PlaygroundCommandRequest, - PlaygroundTarget, -) -from sqlbuild.cli.commands.types import PlaygroundTemplate - - -def resolve_playground_target(*, request: PlaygroundCommandRequest) -> PlaygroundTarget: - """Resolve the playground destination directory and template.""" - - base_dir: Path = request.project_dir if request.project_dir is not None else Path.cwd() - target_dir: Path = Path(request.target_path) - if not target_dir.is_absolute(): - target_dir = base_dir / target_dir - return PlaygroundTarget( - target_dir=target_dir, - template=PlaygroundTemplate(request.template), - ) - - -def write_playground_project(*, target: PlaygroundTarget) -> None: - """Create the playground project files and refresh bundled skills.""" - - create_playground_project(target_dir=target.target_dir, template=target.template.value) - _ = update_sqlbuild_skills(project_dir=_playground_project_dir(target=target)) - - -def _playground_project_dir(*, target: PlaygroundTarget) -> Path: - """Return the SQLBuild project directory inside a generated playground.""" - - if target.template in PLAYGROUND_ORCHESTRATED_TEMPLATES: - return target.target_dir / PLAYGROUND_ORCHESTRATED_PROJECT_DIR - return target.target_dir diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/command.py b/src/sqlbuild/cli/commands/_helpers/refactor/command.py deleted file mode 100644 index 0038433670..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/refactor/command.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Run `sqb rename` and `sqb mv`: plan from compiler facts, verify, then write.""" - -from __future__ import annotations - -import sys -import tempfile -import time -from dataclasses import replace -from pathlib import Path -from typing import TextIO - -from sqlbuild.cli.commands._helpers.refactor.compile_facts import compile_for_refactor -from sqlbuild.cli.commands._helpers.refactor.layer_move import layer_move_suggestion -from sqlbuild.cli.commands._helpers.refactor.native import ( - commit_native_refactor, - migrate_native_refactor, - plan_native_refactor, - stage_native_refactor, -) -from sqlbuild.cli.commands._helpers.refactor.output import ( - render_refactor_json, - render_refactor_text, -) -from sqlbuild.cli.commands._helpers.refactor.target import refactor_request_for_target -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import RefactorCommandRequest, RefactorCompile -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.refactoring.exceptions import RefactorInputError -from sqlbuild.compiler.refactoring.models import RefactorPlan, RefactorRequest -from sqlbuild.compiler.refactoring.types import RefactorOperation, RefactorStatus -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.count_noun import format_count_noun -from sqlbuild.presentation.main.supports_color import supports_color - -_STAGING_PREFIX: str = "sqb-refactor-" - - -def run_refactor_command(*, request: RefactorCommandRequest) -> int: - """Plan, verify in a scratch copy, and write one rename or move.""" - - project_dir: Path = (request.project_dir or Path.cwd()).resolve() - stream: TextIO = sys.stderr if request.json_output else sys.stdout - use_color: bool = not request.no_color and supports_color() - status: TransientStatusReporter = TransientStatusReporter(stream=stream, use_color=use_color) - try: - return _run(request=request, project_dir=project_dir, status=status, use_color=use_color) - except RefactorInputError as error: - status.close() - raise CliUserError(error.message, code=error.code, help=error.help) from error - finally: - status.close() - - -def _run( - *, - request: RefactorCommandRequest, - project_dir: Path, - status: TransientStatusReporter, - use_color: bool, -) -> int: - started: float = time.monotonic() - status.start("Compiling project...") - before: RefactorCompile = compile_for_refactor(project_dir=project_dir) - if before.project is None or before.errors: - status.error("Project does not compile; fix it before renaming (sqb compile)") - _write_errors(errors=before.errors) - return 1 - status.complete(message=f"Compiled project. ({time.monotonic() - started:.2f}s)") - refactor_request: RefactorRequest = refactor_request_for_target( - request=request, all_keys=before.project.graph.all_keys - ) - status.start("Planning edits...") - plan: RefactorPlan - plan_json: str - plan, plan_json = plan_native_refactor(project=before.project, request=refactor_request) - status.complete(message=f"Planned edits to {_files(plan)}.") - if plan.blocking or plan.manual: - status.error("Refused: some references cannot be rewritten safely.") - return _finish( - request=request, - plan=plan, - status=RefactorStatus.REFUSED, - diagnostics=(), - use_color=use_color, - ) - with tempfile.TemporaryDirectory(prefix=_STAGING_PREFIX) as staging: - staging_dir: Path = Path(staging) - status.start("Verifying the edited project compiles...") - originals: dict[str, str] = stage_native_refactor( - project_dir=project_dir, staging_dir=staging_dir, plan_json=plan_json, copy_inputs=True - ) - after: RefactorCompile = compile_for_refactor(project_dir=staging_dir, no_cache=True) - if refactor_request.operation != RefactorOperation.RENAME_COLUMN: - migrated: RefactorPlan - migrated_json: str - migrated, migrated_json = migrate_native_refactor( - plan_json=plan_json, - before=before.project, - after=after.project, - originals=originals, - ) - if migrated.blocking: - status.error("Refused: the renamed model's history cannot be kept.") - return _finish( - request=request, - plan=migrated, - status=RefactorStatus.REFUSED, - diagnostics=(), - use_color=use_color, - ) - if migrated.migrations != plan.migrations: - plan, plan_json = migrated, migrated_json - originals = stage_native_refactor( - project_dir=project_dir, - staging_dir=staging_dir, - plan_json=plan_json, - copy_inputs=False, - ) - after = compile_for_refactor(project_dir=staging_dir, no_cache=True) - diagnostics: tuple[CompilerDiagnostic, ...] = _relative( - errors=after.errors, staging_dir=staging_dir - ) - if diagnostics: - status.error("Verification failed: the edited project does not compile.") - return _finish( - request=request, - plan=plan, - status=RefactorStatus.COMPILE_FAILED, - diagnostics=diagnostics, - use_color=use_color, - ) - status.complete(message="Verified: the edited project compiles.") - if request.dry_run: - return _finish( - request=request, - plan=plan, - status=RefactorStatus.DRY_RUN, - diagnostics=(), - use_color=use_color, - ) - status.start(f"Writing {_files(plan)}...") - commit_native_refactor(project_dir=project_dir, originals=originals, plan_json=plan_json) - status.complete(message=f"Wrote {_files(plan)}.") - return _finish( - request=request, - plan=plan, - status=RefactorStatus.APPLIED, - diagnostics=(), - use_color=use_color, - ) - - -def _finish( - *, - request: RefactorCommandRequest, - plan: RefactorPlan, - status: RefactorStatus, - diagnostics: tuple[CompilerDiagnostic, ...], - use_color: bool, -) -> int: - output: str = ( - render_refactor_json(plan=plan, status=status, diagnostics=diagnostics) - if request.json_output - else render_refactor_text( - plan=plan, - status=status, - diagnostics=diagnostics, - use_color=use_color, - layer_move=layer_move_suggestion( - plan=plan, diagnostics=diagnostics, project_dir=request.project_dir - ), - ) - ) - _ = sys.stdout.write(output) - return 0 if status in {RefactorStatus.APPLIED, RefactorStatus.DRY_RUN} else 1 - - -def _files(plan: RefactorPlan) -> str: - return format_count_noun(count=len(plan.changes), singular="file") - - -def _relative( - *, errors: tuple[CompilerDiagnostic, ...], staging_dir: Path -) -> tuple[CompilerDiagnostic, ...]: - return tuple( - replace(error, path=error.path.relative_to(staging_dir)) - if error.path is not None - and error.path.is_absolute() - and error.path.is_relative_to(staging_dir) - else error - for error in errors - ) - - -def _write_errors(*, errors: tuple[CompilerDiagnostic, ...]) -> None: - error: CompilerDiagnostic - for error in errors: - location: str = f"{error.path.as_posix()}: " if error.path is not None else "" - print(f"error[{error.code}] {location}{error.message}", file=sys.stderr) diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/facts.py b/src/sqlbuild/cli/commands/_helpers/refactor/facts.py deleted file mode 100644 index 743149f0fc..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/refactor/facts.py +++ /dev/null @@ -1,282 +0,0 @@ -"""Plain-data compiler facts for native refactoring; the only `CompiledProject` reader.""" - -from __future__ import annotations - -import json -from collections.abc import Callable -from pathlib import Path - -from sqlbuild._native import find_python_string_locations -from sqlbuild.compiler.compile.models import ( - CompiledSqlExpansion, - ExpansionSpan, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditBlock, - DiscoveredProjectInputs, - DiscoveredSqlTestBlock, -) -from sqlbuild.compiler.refactoring.constants import ( - GENERIC_DIALECT, - MIGRATE_FROM_KEY, - MODEL_KIND_PREFIX, -) -from sqlbuild.compiler.refactoring.models import ManualLocation, RefactorProject, RefactorRequest -from sqlbuild.compiler.refactoring.types import RefactorOperation, SqlFileRole -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move -from sqlbuild.compiler.scopes.main.relocate_declarations_for_move import ( - relocate_declarations_for_move, -) -from sqlbuild.compiler.scopes.models import ( - DeclarationRecord, - MovePreview, - ResourceIdentity, - ScopeDiagnostic, - ScopeIndex, -) -from sqlbuild.compiler.scopes.types import ResourceKind -from sqlbuild.spec.contracts.main.get_config_str import get_config_str -from sqlbuild.spec.contracts.models import SchemaColumn, SourceColumnEntry - -_MATERIALIZED_KEY: str = "materialized" - - -def refactor_facts_json(*, project: RefactorProject, request: RefactorRequest) -> str: - """Return the facts one refactoring plan reads, as the native planner's JSON.""" - - compiled: CompiledProjectView = project.graph.project - discovered: DiscoveredProjectInputs = project.discovered - return json.dumps( - { - "project_dir": str(project.project_dir), - "dialect": compiled.sql_analysis_dialect or GENERIC_DIALECT, - "models": model_facts(project=compiled, project_dir=project.project_dir), - "sources": [ - _columns(name=source.name, columns=source.source_entry.columns) - for source in compiled.sources - ], - "seeds": [ - _columns(name=seed.name, columns=seed.schema_entry.columns) - for seed in compiled.seeds - ], - "model_files": [ - {"path": item.relative_path.as_posix(), "contents": item.contents} - for item in discovered.model_files - ], - "authored_files": _authored_files(discovered=discovered), - "yaml_files": [ - {"path": item.relative_path.as_posix(), "contents": item.contents} - for item in (*discovered.source_files, *discovered.schema_files) - ], - "python_locations": [ - _location(item) for item in _python_locations(project=project, request=request) - ], - } - ) - - -def model_facts_json(*, project: RefactorProject | None) -> str | None: - """Return the model facts of one compiled project as JSON, or None without a project.""" - - if project is None: - return None - return json.dumps(model_facts(project=project.graph.project, project_dir=project.project_dir)) - - -def model_facts(*, project: CompiledProjectView, project_dir: Path) -> list[dict[str, object]]: - """Return every compiled model's refactoring facts.""" - - expansions: dict[Path, CompiledSqlExpansion] = { - path.resolve(): expansion for path, expansion in reversed(project.sql_expansions.items()) - } - return [ - _model(model=model, expansion=expansions.get((project_dir / model.relative_path).resolve())) - for model in project.models - ] - - -def declaration_placement_host(*, project: RefactorProject) -> Callable[[tuple[str, str]], str]: - """Return the callback asking the shared scope placement where moved declarations go.""" - - def placement(move: tuple[str, str]) -> str: - model_name, destination = move - index: ScopeIndex = project.graph.project.scope_index - preview: MovePreview | None - diagnostics: tuple[ScopeDiagnostic, ...] - preview, diagnostics = preview_scope_move( - lookup=build_scope_lookup(index=index), - resource=f"{MODEL_KIND_PREFIX}{model_name}", - destination=destination, - ) - relocated: tuple[DeclarationRecord, ...] | None = ( - None - if preview is None - else relocate_declarations_for_move( - index=index, - resource=ResourceIdentity(kind=ResourceKind.MODEL, name=model_name), - destination=destination, - ) - ) - return json.dumps( - { - "relocated": None - if preview is None or relocated is None - else [ - {"key": repr(record.identity), "path": record.path} - for record in sorted(relocated, key=lambda item: item.identity) - ], - "diagnostics": [item.message for item in diagnostics], - "declarations": [_declaration(record) for record in index.declarations], - } - ) - - return placement - - -def _declaration(record: DeclarationRecord) -> dict[str, object]: - return { - "key": repr(record.identity), - "label": f"{record.identity.kind.value}:{record.identity.name}", - "path": record.path, - "line": record.line, - "column": record.column, - } - - -def _model( - *, model: CompiledModelView, expansion: CompiledSqlExpansion | None -) -> dict[str, object]: - values: dict[str, object] = dict(model.config.values) - return { - "name": model.name, - "path": model.relative_path.as_posix(), - "deps": [{"resource_type": str(dep.resource_type), "name": dep.name} for dep in model.deps], - "query_sql": model.query_sql, - "authored_query_sql": model.authored_query_sql, - "materialized": get_config_str(values=values, key=_MATERIALIZED_KEY), - "declares_migrate_from": MIGRATE_FROM_KEY in values, - "migrate_from_set": values.get(MIGRATE_FROM_KEY) is not None, - "destination": { - "database": model.destination.database, - "schema": model.destination.schema, - "name": model.destination.name, - }, - "inferred_columns": [column.name for column in model.inferred_columns or ()], - "schema_columns": ( - [ - { - "name": column.name, - "migrate_from": column.migrate_from is not None, - "location": ( - { - "path": column.location.path.as_posix(), - "line": column.location.line, - "column": column.location.column, - } - if column.location is not None - else None - ), - } - for column in model.schema_entry.columns - ] - if model.schema_entry is not None - else None - ), - "expansion": ( - { - "expanded_sql": expansion.expanded_sql, - "passes": [_spans(spans=spans) for spans in expansion.passes], - } - if expansion is not None - else None - ), - } - - -def _spans(*, spans: tuple[ExpansionSpan, ...]) -> list[dict[str, int]]: - return [ - { - "source_start": span.source_start, - "source_end": span.source_end, - "output_start": span.output_start, - "output_end": span.output_end, - } - for span in spans - ] - - -def _authored_files(*, discovered: DiscoveredProjectInputs) -> list[dict[str, object]]: - files: list[tuple[SqlFileRole, Path, str, tuple[str, ...]]] = [ - *( - (SqlFileRole.TEST, item.relative_path, item.contents, _block_bodies(item.blocks)) - for item in discovered.test_files - ), - *( - (SqlFileRole.SCENARIO, item.relative_path, item.contents, (item.sql_body,)) - for item in discovered.scenario_files - ), - *( - (SqlFileRole.AUDIT, item.relative_path, item.contents, _block_bodies(item.blocks)) - for item in discovered.audit_files - ), - *( - (SqlFileRole.HOOK, item.relative_path, item.contents, (item.sql_body,)) - for item in discovered.sql_hook_files - ), - *( - (SqlFileRole.FUNCTION, item.relative_path, item.contents, (item.body_sql,)) - for item in discovered.sql_function_files - ), - *( - (SqlFileRole.SCHEMA, item.relative_path, item.contents, ()) - for item in discovered.model_schema_files - ), - ] - return [ - {"role": role.value, "path": path.as_posix(), "contents": contents, "texts": list(texts)} - for role, path, contents, texts in files - ] - - -def _block_bodies( - blocks: tuple[DiscoveredSqlTestBlock | DiscoveredAuditBlock, ...], -) -> tuple[str, ...]: - return tuple(block.sql_body for block in blocks) - - -def _columns( - *, name: str, columns: tuple[SchemaColumn | SourceColumnEntry, ...] -) -> dict[str, object]: - return {"name": name, "columns": [column.name for column in columns]} - - -def _python_locations( - *, project: RefactorProject, request: RefactorRequest -) -> tuple[ManualLocation, ...]: - old: str = request.model_name - if request.operation == RefactorOperation.RENAME_COLUMN: - column: str = request.column_name or "" - return find_python_string_locations( - project_dir=project.project_dir, - discovered=project.discovered, - names=(column,), - reason=f"Python SQL may read column {column} of {old}; check it by hand", - context=old, - ) - return find_python_string_locations( - project_dir=project.project_dir, - discovered=project.discovered, - names=(old,), - reason=f"Python code names model {old}; update it by hand", - ) - - -def _location(location: ManualLocation) -> dict[str, object]: - return { - "path": location.path, - "line": location.line, - "column": location.column, - "reason": location.reason, - } diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/layer_move.py b/src/sqlbuild/cli/commands/_helpers/refactor/layer_move.py deleted file mode 100644 index 6261679987..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/refactor/layer_move.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Suggest the one-step `sqb mv` when a rename fails only on the layer-folder rules.""" - -from __future__ import annotations - -import os -import shlex -import subprocess -from pathlib import Path, PurePosixPath - -from sqlbuild.cli.commands.constants import ( - CONTIGUOUS_LAYER_FOLDERS, - FOLDER_PLACEHOLDER, - LAYER_FOLDER_RULE_CODES, - LAYER_FOLDERS, - MODEL_NAME_LAYER_INDEX, - MODEL_NAME_LAYER_SEPARATOR, - MODEL_NAME_PART_COUNTS, - PROJECT_DIR_OPTION, - SQL_FILE_SUFFIX, - WINDOWS_OS_NAME, -) -from sqlbuild.cli.commands.models import LayerMoveSuggestion -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.refactoring.models import FileChange, RefactorPlan -from sqlbuild.compiler.refactoring.types import RefactorOperation - - -def layer_move_suggestion( - *, - plan: RefactorPlan, - diagnostics: tuple[CompilerDiagnostic, ...], - project_dir: Path | None, - os_name: str = os.name, -) -> LayerMoveSuggestion | None: - """Return the `sqb mv` that renames and moves in one step, or None for any other failure.""" - - renamed: FileChange | None = next((change for change in plan.changes if change.moved), None) - if ( - plan.request.operation != RefactorOperation.RENAME_MODEL - or renamed is None - or not diagnostics - or any( - diagnostic.code not in LAYER_FOLDER_RULE_CODES - or diagnostic.path is None - or diagnostic.path.as_posix() != renamed.path - for diagnostic in diagnostics - ) - ): - return None - new_name: str = plan.request.new_name - new_layer: str | None = _name_layer(new_name) - layer_folder: tuple[str, ...] | None = ( - LAYER_FOLDERS.get(new_layer) if new_layer is not None else None - ) - mirrored: tuple[tuple[str, ...], tuple[str, ...]] | None = ( - _mirrored_folder( - original_path=renamed.original_path, - current_layer=_name_layer(plan.request.model_name), - layer_folder=layer_folder, - ) - if layer_folder is not None - else None - ) - folders: tuple[str, ...] = ( - (*mirrored[0], *mirrored[1]) - if mirrored is not None - else (PurePosixPath(renamed.original_path).parts[0], FOLDER_PLACEHOLDER) - ) - arguments: list[str] = [ - "sqb", - "mv", - plan.request.model_name, - "/".join((*folders, f"{new_name}{SQL_FILE_SUFFIX}")), - ] - if project_dir is not None: - arguments.extend((PROJECT_DIR_OPTION, str(project_dir))) - return LayerMoveSuggestion( - new_name=new_name, - folder="/".join(mirrored[0]) if mirrored is not None else None, - layer_folder="/".join(layer_folder) if layer_folder is not None else None, - command=command_line(arguments=arguments, os_name=os_name), - ) - - -def command_line(*, arguments: list[str], os_name: str) -> str: - """Quote a command for the shell of the platform it is printed on.""" - - if os_name == WINDOWS_OS_NAME: - return subprocess.list2cmdline(arguments) - return shlex.join(arguments) - - -def _name_layer(name: str) -> str | None: - """Return the layer segment of a name in the rule grammar, as the layer-folder rules read it.""" - - parts: list[str] = name.split(MODEL_NAME_LAYER_SEPARATOR) - return parts[MODEL_NAME_LAYER_INDEX] if len(parts) in MODEL_NAME_PART_COUNTS else None - - -def _mirrored_folder( - *, original_path: str, current_layer: str | None, layer_folder: tuple[str, ...] -) -> tuple[tuple[str, ...], tuple[str, ...]] | None: - """Swap the current layer's own folder for the new one, keeping the folders around it.""" - - candidates: tuple[tuple[str, ...], ...] = tuple( - folder - for folder in ( - LAYER_FOLDERS.get(current_layer or ""), - CONTIGUOUS_LAYER_FOLDERS.get(current_layer or ""), - ) - if folder is not None - ) - parent: tuple[str, ...] = PurePosixPath(original_path).parent.parts - for current in candidates: - for start in range(1, len(parent)): - if parent[start : start + len(current)] == current: - return (*parent[:start], *layer_folder), parent[start + len(current) :] - return None diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/native.py b/src/sqlbuild/cli/commands/_helpers/refactor/native.py deleted file mode 100644 index e57c964f73..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/refactor/native.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Plan, stage, migrate and commit one refactoring natively.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.cli.commands._helpers.refactor.facts import ( - declaration_placement_host, - model_facts_json, - refactor_facts_json, -) -from sqlbuild.compiler.refactoring.exceptions import ( - RefactorEditError, - RefactorInputError, - RefactorIOError, - RefactorValueError, - RefactorWriteError, -) -from sqlbuild.compiler.refactoring.models import ( - FileChange, - ManualLocation, - MigrationDeclaration, - RefactorPlan, - RefactorProject, - RefactorRequest, - TextEdit, -) -from sqlbuild.compiler.refactoring.types import ( - EditKind, - NativeRefactorErrorKind, - RefactorOperation, -) - -_PLAN_KEY: str = "plan" -_ERROR_KEY: str = "error" - - -def plan_native_refactor( - *, project: RefactorProject, request: RefactorRequest -) -> tuple[RefactorPlan, str]: - """Plan natively, count its edits, and return the plan with its native JSON.""" - - response: dict[str, Any] = _response( - cast(Any, _native).plan_refactor_json( - refactor_facts_json(project=project, request=request), - json.dumps(_request_payload(request)), - declaration_placement_host(project=project), - ) - ) - payload: dict[str, Any] = _payload(response=response, key=_PLAN_KEY) - plan: RefactorPlan = decode_plan(payload) - return plan, json.dumps(payload) - - -def stage_native_refactor( - *, project_dir: Path, staging_dir: Path, plan_json: str, copy_inputs: bool -) -> dict[str, str]: - """Stage a native plan in the scratch copy and return the original texts.""" - - response: dict[str, Any] = _response( - cast(Any, _native).stage_refactor_plan_json( - str(project_dir), str(staging_dir), plan_json, copy_inputs - ) - ) - return {path: text for path, text in _payload(response=response, key="originals")} - - -def migrate_native_refactor( - *, - plan_json: str, - before: RefactorProject, - after: RefactorProject | None, - originals: dict[str, str], -) -> tuple[RefactorPlan, str]: - """Return the native plan with model `migrate_from` added whenever its relation moves.""" - - before_json: str | None = model_facts_json(project=before) - response: dict[str, Any] = _response( - cast(Any, _native).with_model_migration_json( - plan_json, - before_json, - model_facts_json(project=after), - json.dumps(list(originals.items())), - ) - ) - payload: dict[str, Any] = _payload(response=response, key=_PLAN_KEY) - migrated: RefactorPlan = decode_plan(payload) - return migrated, json.dumps(payload) - - -def commit_native_refactor(*, project_dir: Path, originals: dict[str, str], plan_json: str) -> None: - """Write a verified native plan into the project, restoring every file if a write fails.""" - - _ = _payload( - response=_response( - cast(Any, _native).commit_refactor_plan_json( - str(project_dir), json.dumps(list(originals.items())), plan_json - ) - ), - key="written", - ) - - -def decode_plan(payload: dict[str, Any]) -> RefactorPlan: - """Return the plan dataclasses of one native plan payload.""" - - request: dict[str, Any] = payload["request"] - return RefactorPlan( - request=RefactorRequest( - operation=RefactorOperation(request["operation"]), - model_name=request["model_name"], - new_name=request["new_name"], - column_name=request.get("column_name"), - destination=request.get("destination"), - cascade=bool(request.get("cascade")), - ), - changes=tuple(_change(item) for item in payload["changes"]), - manual=tuple(_manual(item) for item in payload["manual"]), - blocking=tuple(_manual(item) for item in payload["blocking"]), - migrations=tuple( - MigrationDeclaration( - model_name=item["model_name"], - declaration=item["declaration"], - reason=item["reason"], - ) - for item in payload["migrations"] - ), - renamed_columns=tuple( - (str(model), str(old), str(new)) for model, old, new in payload["renamed_columns"] - ), - help=payload.get("help"), - ) - - -def _change(item: dict[str, Any]) -> FileChange: - return FileChange( - path=item["path"], - original_path=item["original_path"], - edits=tuple(_edit(edit) for edit in item["edits"]), - ) - - -def _edit(item: dict[str, Any]) -> TextEdit: - return TextEdit( - start=int(item["start"]), - end=int(item["end"]), - replacement=item["replacement"], - kind=EditKind(item["kind"]), - line=int(item["line"]), - column=int(item["column"]), - before=item["before"], - after=item["after"], - ) - - -def _manual(item: dict[str, Any]) -> ManualLocation: - return ManualLocation( - path=item["path"], line=item.get("line"), column=item.get("column"), reason=item["reason"] - ) - - -def _request_payload(request: RefactorRequest) -> dict[str, object]: - return { - "operation": request.operation.value, - "model_name": request.model_name, - "new_name": request.new_name, - "column_name": request.column_name, - "destination": request.destination, - "cascade": request.cascade, - } - - -def _response(text: str) -> dict[str, Any]: - return cast(dict[str, Any], json.loads(text)) - - -def _payload(*, response: dict[str, Any], key: str) -> Any: - error: dict[str, Any] | None = response.get(_ERROR_KEY) - if error is None: - return response[key] - kind: NativeRefactorErrorKind = NativeRefactorErrorKind(str(error["kind"])) - message: str = str(error["message"]) - if kind is NativeRefactorErrorKind.VALUE: - raise RefactorValueError(message) - if kind is NativeRefactorErrorKind.IO: - raise RefactorIOError(message) - if kind is NativeRefactorErrorKind.EDIT: - raise RefactorEditError(message) - if kind is NativeRefactorErrorKind.WRITE: - raise RefactorWriteError(message, help=error.get("help")) - raise RefactorInputError(message, code=str(error["code"]), help=error.get("help")) diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/output.py b/src/sqlbuild/cli/commands/_helpers/refactor/output.py deleted file mode 100644 index 55d52bfb42..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/refactor/output.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Human and JSON output for `sqb rename` and `sqb mv`.""" - -from __future__ import annotations - -import json - -from sqlbuild.cli.commands.constants import FOLDER_PLACEHOLDER -from sqlbuild.cli.commands.models import LayerMoveSuggestion -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.refactoring.constants import OPERATION_TITLES -from sqlbuild.compiler.refactoring.models import ( - FileChange, - ManualLocation, - MigrationDeclaration, - RefactorPlan, - TextEdit, -) -from sqlbuild.compiler.refactoring.types import RefactorOperation, RefactorStatus -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.count_noun import format_count_noun -from sqlbuild.presentation.main.tree_connector import tree_connector - -_CONTINUE: str = "\u2502 " -_BLANK: str = " " -_POSITION_WIDTH: int = 8 -_KIND_WIDTH: int = 10 -_EXCERPT_LIMIT: int = 60 - - -def render_refactor_text( - *, - plan: RefactorPlan, - status: RefactorStatus, - diagnostics: tuple[CompilerDiagnostic, ...], - use_color: bool, - layer_move: LayerMoveSuggestion | None = None, -) -> str: - """Render the edit tree, manual locations, and one terminal status line.""" - - style: CliStyle = CliStyle(use_color=use_color) - lines: list[str] = [style.title(_heading(plan))] - blocked: bool = status == RefactorStatus.REFUSED - if not blocked or not (plan.blocking or plan.manual): - lines.extend(_change_lines(style=style, changes=plan.changes)) - if plan.migrations and not blocked: - lines.append(style.section("Migrations")) - lines.extend(_migration_lines(style=style, migrations=plan.migrations)) - if blocked: - lines.append(style.error_strong(f"Cannot {_verb(plan)} {_target(plan)} automatically")) - lines.extend(_location_lines(style=style, locations=(*plan.blocking, *plan.manual))) - if plan.help and plan.manual: - lines.append(plan.help) - if diagnostics: - lines.append(style.error_strong("Compile errors in the edited project")) - lines.extend(_diagnostic_lines(style=style, diagnostics=diagnostics)) - if layer_move is not None: - lines.extend(_layer_move_lines(style=style, layer_move=layer_move)) - lines.append(_status_line(style=style, plan=plan, status=status)) - return "\n".join(lines) + "\n" - - -def render_refactor_json( - *, - plan: RefactorPlan, - status: RefactorStatus, - diagnostics: tuple[CompilerDiagnostic, ...], -) -> str: - """Render the same facts as stable JSON.""" - - payload: dict[str, object] = { - "operation": plan.request.operation.value, - "target": _target(plan), - "new_name": plan.request.new_name, - "destination": plan.request.destination, - "cascade": plan.request.cascade, - "status": status.value, - "files_changed": len(plan.changes) if status == RefactorStatus.APPLIED else 0, - "files": [_file_payload(change) for change in plan.changes], - "migrations": [ - { - "model": migration.model_name, - "declaration": migration.declaration, - "reason": migration.reason, - } - for migration in plan.migrations - ], - "renamed_columns": [ - {"model": model, "from": old, "to": new} for model, old, new in plan.renamed_columns - ], - "manual": [_location_payload(item) for item in plan.manual], - "blocking": [_location_payload(item) for item in plan.blocking], - "compile": { - "ok": not diagnostics, - "errors": [ - { - "code": diagnostic.code, - "message": diagnostic.message, - "path": diagnostic.path.as_posix() if diagnostic.path is not None else None, - "line": diagnostic.line, - } - for diagnostic in diagnostics - ], - }, - } - return json.dumps(payload, indent=2) + "\n" - - -def _file_payload(change: FileChange) -> dict[str, object]: - return { - "path": change.path, - "from_path": change.original_path if change.moved else None, - "edits": [ - _edit_payload(edit) for edit in sorted(change.edits, key=lambda item: item.start) - ], - } - - -def _edit_payload(edit: TextEdit) -> dict[str, object]: - return { - "line": edit.line, - "column": edit.column, - "kind": edit.kind.value, - "before": edit.before, - "after": edit.after, - } - - -def _heading(plan: RefactorPlan) -> str: - title: str = OPERATION_TITLES[plan.request.operation] - if plan.request.operation == RefactorOperation.RENAME_COLUMN: - return ( - f"{title} {plan.request.model_name}.{plan.request.column_name} -> " - f"{plan.request.new_name}" - ) - if plan.request.operation == RefactorOperation.MOVE_MODEL: - return f"{title} {plan.request.model_name} -> {plan.request.destination}" - return f"{title} {plan.request.model_name} -> {plan.request.new_name}" - - -def _verb(plan: RefactorPlan) -> str: - return "move" if plan.request.operation == RefactorOperation.MOVE_MODEL else "rename" - - -def _target(plan: RefactorPlan) -> str: - if plan.request.operation == RefactorOperation.RENAME_COLUMN: - return f"column:{plan.request.model_name}.{plan.request.column_name}" - return f"model:{plan.request.model_name}" - - -def _change_lines(*, style: CliStyle, changes: tuple[FileChange, ...]) -> list[str]: - lines: list[str] = [] - index: int - change: FileChange - for index, change in enumerate(changes): - last: bool = index == len(changes) - 1 - moved: str = style.muted(f" (moved from {change.original_path})") if change.moved else "" - lines.append(f"{tree_connector(style=style, last=last)} {change.path}{moved}") - prefix: str = _BLANK if last else _CONTINUE - edits: list[TextEdit] = sorted(change.edits, key=lambda item: item.start) - edit_index: int - edit: TextEdit - for edit_index, edit in enumerate(edits): - connector: str = tree_connector(style=style, last=edit_index == len(edits) - 1) - position: str = f"{edit.line}:{edit.column}".ljust(_POSITION_WIDTH) - lines.append( - f"{style.muted(prefix)}{connector} {position}{edit.kind.value.ljust(_KIND_WIDTH)}" - f"{_excerpt(edit.before)} -> {_excerpt(edit.after)}" - if edit.before - else f"{style.muted(prefix)}{connector} {position}" - f"{edit.kind.value.ljust(_KIND_WIDTH)}+ {_excerpt(edit.after)}" - ) - return lines - - -def _migration_lines(*, style: CliStyle, migrations: tuple[MigrationDeclaration, ...]) -> list[str]: - return [ - f"{tree_connector(style=style, last=index == len(migrations) - 1)} " - f"{style.object_name(item.model_name)} {item.declaration} {style.muted(item.reason)}" - for index, item in enumerate(migrations) - ] - - -def _location_lines(*, style: CliStyle, locations: tuple[ManualLocation, ...]) -> list[str]: - return [ - f"{tree_connector(style=style, last=index == len(locations) - 1)} " - f"{_position(item)} {item.reason}" - for index, item in enumerate(locations) - ] - - -def _diagnostic_lines(*, style: CliStyle, diagnostics: tuple[CompilerDiagnostic, ...]) -> list[str]: - return [ - f"{tree_connector(style=style, last=index == len(diagnostics) - 1)} " - f"{diagnostic.code} " - + ( - f"{diagnostic.path.as_posix()}:{diagnostic.line or ''} " - if diagnostic.path is not None - else "" - ) - + diagnostic.message.splitlines()[0] - for index, diagnostic in enumerate(diagnostics) - ] - - -def _layer_move_lines(*, style: CliStyle, layer_move: LayerMoveSuggestion) -> list[str]: - folder: str = ( - f"under {layer_move.folder}/" - if layer_move.folder is not None - else f"under a layer folder named {layer_move.layer_folder}/" - if layer_move.layer_folder is not None - else "under another layer's folder" - ) - note: str = ( - "(change the folder if you want it somewhere else)" - if layer_move.folder is not None - else f"(replace {FOLDER_PLACEHOLDER}/ with the folder the model should live in)" - ) - return [ - f"{style.object_name(layer_move.new_name)} belongs {folder}", - " Rename and move in one step:", - f" {style.command(layer_move.command)}", - f" {style.muted(note)}", - ] - - -def _status_line(*, style: CliStyle, plan: RefactorPlan, status: RefactorStatus) -> str: - files: str = format_count_noun(count=len(plan.changes), singular="file") - if status == RefactorStatus.APPLIED: - return style.success_strong(f"Compiled: ok, {files} changed") - if status == RefactorStatus.DRY_RUN: - return style.success_strong(f"Dry run: compiles, {files} would change; nothing written") - if status == RefactorStatus.COMPILE_FAILED: - return style.error_strong("Compiled: failed; no files changed") - return style.error_strong("Refused; no files changed") - - -def _position(location: ManualLocation) -> str: - if location.line is None: - return location.path - return f"{location.path}:{location.line}:{location.column or 1}" - - -def _location_payload(location: ManualLocation) -> dict[str, object]: - return { - "path": location.path, - "line": location.line, - "column": location.column, - "reason": location.reason, - } - - -def _excerpt(text: str) -> str: - single: str = " ".join(text.split()) - return single if len(single) <= _EXCERPT_LIMIT else single[: _EXCERPT_LIMIT - 1] + "\u2026" diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/target.py b/src/sqlbuild/cli/commands/_helpers/refactor/target.py deleted file mode 100644 index 19a78f0325..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/refactor/target.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Read the target of `sqb rename` and `sqb mv` the way `sqb lineage` reads targets.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.lineage.selection import resolve_lineage_target -from sqlbuild.cli.commands.constants import COLUMN_TARGET_KIND, COLUMN_TARGET_SEPARATOR -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import LineageTarget, RefactorCommandRequest -from sqlbuild.cli.commands.types import CliCommand -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.refactoring.models import RefactorRequest -from sqlbuild.compiler.refactoring.types import RefactorOperation - -_RENAME_USAGE: str = ( - "write the target as or .; the model: and column: prefixes are optional" -) -_MOVE_USAGE: str = "write the target as ; the model: prefix is optional" -_RENAME_SCOPE: str = "sqb rename renames models and model columns" -_MOVE_SCOPE: str = "sqb mv moves models" - - -def refactor_request_for_target( - *, request: RefactorCommandRequest, all_keys: dict[str, CompiledObjectKey] -) -> RefactorRequest: - """Read a `sqb rename` or `sqb mv` target the way `sqb lineage` reads its targets.""" - - raw: str = request.target.strip() - target: LineageTarget = resolve_lineage_target(all_keys=all_keys, target=raw) - is_move: bool = request.command == CliCommand.MV - _check_target(raw=raw, target=target, is_move=is_move) - model_name: str = target.name.partition(COLUMN_TARGET_SEPARATOR)[0] - if is_move: - return RefactorRequest( - operation=RefactorOperation.MOVE_MODEL, - model_name=model_name, - new_name="", - destination=request.destination or "", - ) - new_name: str = (request.new_name or "").strip() - if target.column_name is None: - if request.cascade: - raise CliUserError("--cascade applies to column renames only", code="C954") - return RefactorRequest( - operation=RefactorOperation.RENAME_MODEL, - model_name=model_name, - new_name=new_name, - ) - return RefactorRequest( - operation=RefactorOperation.RENAME_COLUMN, - model_name=model_name, - column_name=target.column_name, - new_name=new_name, - cascade=request.cascade, - ) - - -def _check_target(*, raw: str, target: LineageTarget, is_move: bool) -> None: - verb: str = "move" if is_move else "rename" - usage: str = _MOVE_USAGE if is_move else _RENAME_USAGE - key: CompiledObjectKey | None = target.key - if not target.name or (target.column_name is not None and not target.column_name): - raise CliUserError(f"cannot {verb} '{raw}'", code="C954", help=usage) - if key is not None and key.resource_type != CompiledResourceType.MODEL: - raise CliUserError( - f"cannot {verb} '{raw}': {key.name} is a {key.resource_type}; " - f"{_MOVE_SCOPE if is_move else _RENAME_SCOPE}", - code="C954", - help=usage, - ) - if key is not None and not target.kind_matches: - found: str = ( - f"column: needs ., and {key.name} names a model" - if target.kind == COLUMN_TARGET_KIND - else f"{key.name} is a {key.resource_type}, not a {target.kind}" - ) - raise CliUserError(f"cannot {verb} '{raw}': {found}", code="C954", help=usage) - if is_move and target.column_name is not None: - raise CliUserError( - f"sqb mv moves models; '{raw}' names a column", - code="C954", - help=f"rename a column with sqb rename {target.name} ", - ) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py b/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py index 9127cfd8a9..ccb309ab74 100644 --- a/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py +++ b/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py @@ -4,9 +4,9 @@ from pathlib import Path +from sqlbuild._native import resolve_project_connection_config from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import resolve_project_connection_config from sqlbuild.cli.commands.models import AdapterConnectionContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/connection.py b/src/sqlbuild/cli/commands/_helpers/runtime/connection.py deleted file mode 100644 index 1e9ed21167..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/connection.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Connection config resolution for CLI commands.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.cli.commands._helpers.runtime.warehouses import apply_command_warehouse -from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config -from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.main.profile.profile_connection import ( - resolve_raw_dbt_profile_connection, -) -from sqlbuild.integrations.dbt.models import NormalizedDbtProfileConnection -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, TargetConfig - -_DUCKDB_SNOWFLAKE_LIKE_WARNING_KEYS: frozenset[str] = frozenset( - {"account", "authenticator", "role", "token", "warehouse"} -) -_DBT_PROFILE_CONNECTION_ROUTING_KEYS: frozenset[str] = frozenset( - {"source", "profile", "target", "project_dir", "profiles_dir"} -) -_CONNECTION_SOURCE_KEY: str = "source" -_DBT_PROFILE_CONNECTION_SOURCE: str = "dbt_profile" -_DATABASE_KEY: str = "database" -_DUCKDB_MEMORY_DATABASE: str = ":memory:" - - -def resolve_connection_config( - *, - raw_config: dict[str, object], - project_dir: Path, - adapter_name: str, - discovered_inputs: DiscoveredProjectInputs | None = None, - cli_vars: dict[str, object] | None = None, - selected_target: str | None = None, -) -> dict[str, object]: - """Resolve dbt profiles, relative DuckDB paths, and the command group's warehouse.""" - - config: dict[str, object] = dict(raw_config) - if discovered_inputs is not None: - resolved_dbt_profile: NormalizedDbtProfileConnection | None = ( - resolve_raw_dbt_profile_connection( - raw_config=config, - project_dir=project_dir, - adapter_name=adapter_name, - project_config=discovered_inputs.project_config, - cli_vars=cli_vars, - ) - ) - if resolved_dbt_profile is not None: - for warning in resolved_dbt_profile.warnings: - print(f"Warning: {warning}", file=sys.stderr) - user_overrides: dict[str, object] = { - key: value - for key, value in raw_config.items() - if key not in _DBT_PROFILE_CONNECTION_ROUTING_KEYS - } - config = {**resolved_dbt_profile.connection, **user_overrides} - elif config.get(_CONNECTION_SOURCE_KEY) == _DBT_PROFILE_CONNECTION_SOURCE: - raise DbtProfileError("connection.source = 'dbt_profile' requires project configuration") - _warn_if_duckdb_has_snowflake_like_keys(adapter_name=adapter_name, config=config) - database: object | None = config.get(_DATABASE_KEY) - if ( - adapter_name == BuiltinAdapter.DUCKDB - and isinstance(database, str) - and not Path(database).is_absolute() - and database != _DUCKDB_MEMORY_DATABASE - ): - config[_DATABASE_KEY] = str(project_dir / database) - return apply_command_warehouse( - config=config, - adapter_name=adapter_name, - project_dir=project_dir, - discovered_inputs=discovered_inputs, - selected_target=selected_target, - ) - - -def _warn_if_duckdb_has_snowflake_like_keys( - *, adapter_name: str, config: dict[str, object] -) -> None: - if adapter_name != BuiltinAdapter.DUCKDB: - return - suspicious_keys: tuple[str, ...] = tuple( - sorted(key for key in config if key in _DUCKDB_SNOWFLAKE_LIKE_WARNING_KEYS) - ) - if not suspicious_keys: - return - key_list: str = ", ".join(suspicious_keys) - print( - "Warning: DuckDB adapter is active, but connection contains " - f"Snowflake-like keys: {key_list}. If this is a Snowflake local config, " - 'add `adapter = "snowflake"` at the top of sqlbuild_local.toml, before any section.', - file=sys.stderr, - ) - - -def resolve_project_connection_config( - *, - discovered_inputs: DiscoveredProjectInputs, - project_dir: Path, - selected_target: str | None = None, - cli_vars: dict[str, object] | None = None, - require_connection: bool = True, -) -> dict[str, object]: - """Resolve the project connection; offline modes pass ``require_connection=False``.""" - - if require_connection: - _require_configured_connection( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - selected_target=selected_target, - ) - return resolve_connection_config( - raw_config=build_effective_connection_config( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - cli_vars=cli_vars, - ), - project_dir=project_dir, - adapter_name=resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - discovered_inputs=discovered_inputs, - cli_vars=cli_vars, - selected_target=selected_target, - ) - - -def resolve_target_connection_config( - *, - discovered_inputs: DiscoveredProjectInputs, - project_dir: Path, - target_name: str, - cli_vars: dict[str, object] | None = None, -) -> dict[str, object]: - """Resolve the effective connection config for one named target.""" - - return resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - selected_target=target_name, - cli_vars=cli_vars, - ) - - -def _require_configured_connection( - *, - discovered_inputs: DiscoveredProjectInputs, - project_dir: Path, - selected_target: str | None, -) -> None: - """Reject warehouse commands whose connection would come from no configuration at all.""" - - project_config: ProjectConfig = discovered_inputs.project_config - local_config: LocalConfig = discovered_inputs.local_config - if project_config.connection or local_config.connection: - return - target_name: str | None = resolve_target_name( - project_config=project_config, - local_config=local_config, - selected_target=selected_target, - ) - if target_name is None: - raise ProjectConfigError( - f"{project_dir / PROJECT_CONFIG_FILENAME} no target is selected and no top-level " - "[connection] is configured; add [connections.] and a [targets.] that " - "uses it, then set default_target or pass --target" - ) - target_config: TargetConfig = resolve_target_config( - project_config=project_config, - local_config=local_config, - target_name=target_name, - ) - if target_config.connection_name is not None or target_config.connection: - return - raise ProjectConfigError( - f"{project_dir / PROJECT_CONFIG_FILENAME} target {target_name} has no connection; add " - f'[connections.] (or set connection = "" on [targets.{target_name}]) in ' - "sqlbuild_project.toml or sqlbuild_local.toml" - ) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/preview_target.py b/src/sqlbuild/cli/commands/_helpers/runtime/preview_target.py deleted file mode 100644 index 66e1278b31..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/preview_target.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Shared validation for inspection-only ``--as`` target previews.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - - -def validate_preview_target( - *, discovered_inputs: DiscoveredProjectInputs, as_target: str | None, command_name: str -) -> None: - """Fail clearly when ``--as`` names a target that is not configured.""" - - if as_target is None: - return - configured: set[str] = set(discovered_inputs.project_config.targets) | set( - discovered_inputs.local_config.targets - ) - if as_target not in configured: - raise CliUserError( - f"unknown target '{as_target}' for {command_name} --as", - help=f"Configured targets: {', '.join(sorted(configured)) or 'none'}.", - ) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/warehouses.py b/src/sqlbuild/cli/commands/_helpers/runtime/warehouses.py deleted file mode 100644 index cca799a06b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/warehouses.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Command-group warehouse selection for CLI warehouse connections.""" - -from __future__ import annotations - -import re -from pathlib import Path - -from sqlbuild.adapter.discovery.main.session_warehouse_support import ( - adapter_supports_session_warehouse, -) -from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import ( - current_command_warehouse_scope, -) -from sqlbuild.cli.commands.constants import ( - CLI_WAREHOUSE_SOURCE, - CONNECTION_DEFAULT_WAREHOUSE_SOURCE, - CONNECTION_WAREHOUSE_SOURCE, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.entry.models import ( - AuthoredTargetWarehouse, - CommandWarehouseScope, - ResolvedWarehouse, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config -from sqlbuild.compiler.compile.main.expand_template_data import expand_template_data -from sqlbuild.compiler.discovery.constants import LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME -from sqlbuild.compiler.discovery.main.discover_configuration import ( - discover_project_configuration, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.spec.contracts.constants import CONNECTION_WAREHOUSE_KEY -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import LocalTargetConfig, TargetConfig -from sqlbuild.spec.contracts.types import WarehouseGroup - -_SNOWFLAKE_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile( - r'[A-Za-z_][A-Za-z0-9_$]*|"(?:[^"]|"")+"' -) -_CONNECTION_SOURCES: frozenset[str] = frozenset( - {CONNECTION_WAREHOUSE_SOURCE, CONNECTION_DEFAULT_WAREHOUSE_SOURCE} -) - - -def validate_cli_warehouse(*, cli_warehouse: str | None, project_dir: Path) -> None: - """Reject an empty, malformed, or unsupported `--warehouse` before the command runs.""" - - if cli_warehouse is None: - return - _validate_cli_warehouse_identifier(cli_warehouse) - discovered: DiscoveredProjectInputs = discover_project_configuration(project_dir=project_dir) - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered.project_config, local_config=discovered.local_config - ) - if not adapter_supports_session_warehouse(adapter_name=adapter_name, project_dir=project_dir): - _raise_unsupported_cli_warehouse(cli_warehouse=cli_warehouse, adapter_name=adapter_name) - - -def find_target_group_warehouse( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - group: WarehouseGroup, -) -> AuthoredTargetWarehouse | None: - """Find the active target's authored, unexpanded warehouse for one command group.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - if target_name is None: - return None - local_target: LocalTargetConfig | None = discovered_inputs.local_config.targets.get(target_name) - project_target: TargetConfig | None = discovered_inputs.project_config.targets.get(target_name) - local_value: str | None = ( - None if local_target is None else local_target.warehouses.for_group(group) - ) - project_value: str | None = ( - None if project_target is None else project_target.warehouses.for_group(group) - ) - authored: str | None = local_value if local_value is not None else project_value - if authored is None: - return None - return AuthoredTargetWarehouse( - warehouse=authored, - target_name=target_name, - group=group, - file_name=LOCAL_CONFIG_FILENAME if local_value is not None else PROJECT_CONFIG_FILENAME, - ) - - -def resolve_command_warehouse( - *, - discovered_inputs: DiscoveredProjectInputs | None, - selected_target: str | None, - group: WarehouseGroup | None, - cli_warehouse: str | None, - connection_config: dict[str, object], - cli_vars: dict[str, object] | None = None, -) -> ResolvedWarehouse: - """Resolve one group's warehouse: CLI, then target group, then connection, then default.""" - - if cli_warehouse is not None: - _validate_cli_warehouse_identifier(cli_warehouse) - return ResolvedWarehouse(warehouse=cli_warehouse, source=CLI_WAREHOUSE_SOURCE) - authored: AuthoredTargetWarehouse | None = ( - None - if group is None or discovered_inputs is None - else find_target_group_warehouse( - discovered_inputs=discovered_inputs, selected_target=selected_target, group=group - ) - ) - if authored is not None and discovered_inputs is not None: - return _expand_target_group_warehouse( - authored=authored, - discovered_inputs=discovered_inputs, - selected_target=selected_target, - cli_vars=cli_vars, - ) - connection_warehouse: object | None = connection_config.get(CONNECTION_WAREHOUSE_KEY) - if isinstance(connection_warehouse, str) and connection_warehouse.strip(): - return ResolvedWarehouse(warehouse=connection_warehouse, source=CONNECTION_WAREHOUSE_SOURCE) - return ResolvedWarehouse(warehouse=None, source=CONNECTION_DEFAULT_WAREHOUSE_SOURCE) - - -def apply_command_warehouse( - *, - config: dict[str, object], - adapter_name: str, - project_dir: Path | None, - discovered_inputs: DiscoveredProjectInputs | None, - selected_target: str | None, -) -> dict[str, object]: - """Return connection config whose warehouse follows the active command's group.""" - - scope: CommandWarehouseScope = current_command_warehouse_scope() - authored: AuthoredTargetWarehouse | None = ( - None - if scope.group is None or discovered_inputs is None - else find_target_group_warehouse( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - group=scope.group, - ) - ) - if scope.cli_warehouse is None and authored is None: - return config - if not adapter_supports_session_warehouse(adapter_name=adapter_name, project_dir=project_dir): - if scope.cli_warehouse is not None: - _raise_unsupported_cli_warehouse( - cli_warehouse=scope.cli_warehouse, adapter_name=adapter_name - ) - return config - resolved: ResolvedWarehouse = resolve_command_warehouse( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - group=scope.group, - cli_warehouse=scope.cli_warehouse, - connection_config=config, - cli_vars=scope.cli_vars, - ) - if resolved.source in _CONNECTION_SOURCES: - return config - return {**config, CONNECTION_WAREHOUSE_KEY: resolved.warehouse} - - -def describe_connection_warehouse( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - connection_config: dict[str, object], -) -> ResolvedWarehouse: - """Describe the warehouse an already resolved connection uses and where it came from.""" - - scope: CommandWarehouseScope = current_command_warehouse_scope() - connection_warehouse: object | None = connection_config.get(CONNECTION_WAREHOUSE_KEY) - if not isinstance(connection_warehouse, str) or not connection_warehouse.strip(): - return ResolvedWarehouse(warehouse=None, source=CONNECTION_DEFAULT_WAREHOUSE_SOURCE) - if scope.cli_warehouse == connection_warehouse: - return ResolvedWarehouse(warehouse=connection_warehouse, source=CLI_WAREHOUSE_SOURCE) - authored: AuthoredTargetWarehouse | None = ( - None - if scope.group is None - else find_target_group_warehouse( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - group=scope.group, - ) - ) - if authored is not None and _expands_to( - authored=authored, - warehouse=connection_warehouse, - discovered_inputs=discovered_inputs, - selected_target=selected_target, - cli_vars=scope.cli_vars, - ): - return ResolvedWarehouse(warehouse=connection_warehouse, source=authored.source) - return ResolvedWarehouse(warehouse=connection_warehouse, source=CONNECTION_WAREHOUSE_SOURCE) - - -def _expands_to( - *, - authored: AuthoredTargetWarehouse, - warehouse: str, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - cli_vars: dict[str, object], -) -> bool: - try: - resolved: ResolvedWarehouse = _expand_target_group_warehouse( - authored=authored, - discovered_inputs=discovered_inputs, - selected_target=selected_target, - cli_vars=cli_vars, - ) - except (CompileInputError, CliUserError): - return False - return resolved.warehouse == warehouse - - -def _expand_target_group_warehouse( - *, - authored: AuthoredTargetWarehouse, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - cli_vars: dict[str, object] | None, -) -> ResolvedWarehouse: - effective_vars: dict[str, object] = build_effective_runtime_config( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - cli_vars=cli_vars, - )[1] - warehouse: str = str( - expand_template_data( - value=authored.warehouse, - variables=effective_vars, - context_values={}, - context_label=f"targets.{authored.target_name}.warehouses.{authored.group.value}", - allow_context=False, - preserve_context_tokens=False, - preserve_unknown_context=False, - ) - ) - if _SNOWFLAKE_IDENTIFIER_PATTERN.fullmatch(warehouse) is None: - section: str = f"targets.{authored.target_name}.warehouses" - raise CliUserError( - f"{authored.file_name} [{section}] {authored.group.value} resolves to " - f"{warehouse!r}, which is not a valid Snowflake warehouse identifier", - code="C261", - help=setting_help( - purpose='to use an unquoted identifier (or a double-quoted one such as "Adhoc WH")', - file_name=authored.file_name, - section=section, - key=authored.group.value, - value="ADHOC_WH", - ), - ) - return ResolvedWarehouse(warehouse=warehouse, source=authored.source) - - -def _validate_cli_warehouse_identifier(cli_warehouse: str) -> None: - if not cli_warehouse.strip(): - raise CliUserError( - "--warehouse requires a warehouse name", - code="C261", - help="pass a name such as `--warehouse ADHOC_WH`, or omit the flag", - ) - if _SNOWFLAKE_IDENTIFIER_PATTERN.fullmatch(cli_warehouse) is None: - raise CliUserError( - f"--warehouse {cli_warehouse!r} is not a valid Snowflake warehouse identifier", - code="C261", - help=( - "pass an unquoted identifier such as `--warehouse ADHOC_WH`, or a double-quoted " - """identifier such as `--warehouse '"Adhoc WH"'`""" - ), - ) - - -def _raise_unsupported_cli_warehouse(*, cli_warehouse: str, adapter_name: str) -> None: - raise CliUserError( - f"--warehouse {cli_warehouse} was given, but the '{adapter_name}' adapter has no " - "warehouse to select", - code="C260", - help=( - "remove --warehouse; it needs an adapter with a session warehouse, such as " - "snowflake or an adapter extending it" - ), - ) diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py index 9b6c7ff7f1..1e608b5522 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py @@ -6,15 +6,18 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import discover_project_inputs, run_compile_only_pipeline +from sqlbuild._native import ( + PlanningProgressReporter, + discover_project_inputs, + resolve_project_connection_config, + run_compile_only_pipeline, + write_execution_header, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, -) from sqlbuild.cli.commands._helpers.scenario_capture.capture_run import ( build_scenario_capture_settings, run_scenario_capture_run, @@ -35,8 +38,6 @@ ScenarioRunOutputContext, ScenarioSnapshotLimitInputs, ) -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter -from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py index d47847c33c..2609f40093 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import ConnectionProgressReporter, format_summary_footer from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.scenario_output.namespace import ( scenario_activity_message, @@ -15,7 +16,6 @@ from sqlbuild.cli.commands._helpers.scenario_output.result_output import render_result_error from sqlbuild.cli.commands.constants import SUCCESS_STATUS from sqlbuild.cli.commands.models import ScenarioRunOutputContext -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ScenarioExecutionPlan @@ -31,7 +31,6 @@ ) from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.summary_footer import format_summary_footer from sqlbuild.runtime.contracts.models import ConnectionHooks _SCENARIO_NAME_WIDTH: int = 64 diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py index f396216c5d..cfece81eda 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py @@ -7,9 +7,14 @@ from typing import TextIO from sqlbuild._native import ( + ConnectionProgressReporter, + PlanningProgressReporter, discover_project_inputs, format_scenario_snapshot_execution_json, + format_summary_footer, + resolve_project_connection_config, run_compile_only_pipeline, + write_execution_header, write_execution_json_output, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -18,9 +23,6 @@ resolve_external_sql_reference_resolver, ) from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, -) from sqlbuild.cli.commands._helpers.scenario_capture.capture_run import ( build_scenario_capture_settings, ) @@ -51,9 +53,6 @@ ScenarioSnapshotLimitInputs, ScenarioTestCommandRequest, ) -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter -from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance @@ -75,7 +74,6 @@ ) from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.summary_footer import format_summary_footer from sqlbuild.presentation.main.supports_color import supports_color from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py index 5e9232ad5b..b7eb01a152 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py @@ -4,6 +4,7 @@ from pathlib import Path +from sqlbuild._native import ConnectionProgressReporter from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.scenario_output.result_output import complete_scenario_run from sqlbuild.cli.commands._helpers.scenario_output.run_presentation import ( @@ -12,7 +13,6 @@ ) from sqlbuild.cli.commands.constants import SUCCESS_STATUS from sqlbuild.cli.commands.models import ScenarioRunOutputContext, ScenarioRunPresentation -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.pipeline.main.run import run_scenario_test_pipeline diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_output/run_presentation.py b/src/sqlbuild/cli/commands/_helpers/scenario_output/run_presentation.py index 3ed296813f..f231b54ecf 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_output/run_presentation.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_output/run_presentation.py @@ -4,7 +4,11 @@ import time -from sqlbuild._native import format_scenario_execution_json, write_execution_json_output +from sqlbuild._native import ( + format_scenario_execution_json, + format_summary_footer, + write_execution_json_output, +) from sqlbuild.cli.commands._helpers.scenario_output.namespace import ( scenario_activity_message, write_namespace_completion, @@ -13,7 +17,6 @@ from sqlbuild.executor.scenario.models import ScenarioRunResult from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.summary_footer import format_summary_footer def begin_scenario_run( diff --git a/src/sqlbuild/cli/commands/_helpers/scope/__init__.py b/src/sqlbuild/cli/commands/_helpers/scope/__init__.py deleted file mode 100644 index 3c9d0d5504..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scope/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Scope command helpers.""" diff --git a/src/sqlbuild/cli/commands/_helpers/scope/command.py b/src/sqlbuild/cli/commands/_helpers/scope/command.py deleted file mode 100644 index 0bd6ae33d6..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scope/command.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Offline declaration-scope inspection command.""" - -from __future__ import annotations - -import sys -from dataclasses import replace -from pathlib import Path -from typing import TextIO - -import sqlbuild._native as _native -from sqlbuild.cli.commands._helpers.scope.output import render_scope_result -from sqlbuild.cli.commands._helpers.scope.target import qualify_scope_target -from sqlbuild.cli.commands.models import ScopeCommandRequest -from sqlbuild.cli.commands.types import ScopeIndexLoader -from sqlbuild.compiler.scopes.main.browse_scope_folders import browse_scope_folders -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.main.explain_scope_declaration import explain_scope_declaration -from sqlbuild.compiler.scopes.main.list_scope_declarations import list_scope_declarations -from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move -from sqlbuild.compiler.scopes.main.query_scope_report import query_scope_report -from sqlbuild.compiler.scopes.main.serialize_scope_report import serialize_scope_report -from sqlbuild.compiler.scopes.models import ( - ScopeBrowseResult, - ScopeDiagnostic, - ScopeIndex, - ScopeListResult, - ScopeLookup, - ScopeReport, - ScopeReportFilters, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, DiagnosticSeverity -from sqlbuild.presentation.main.supports_color import supports_color - - -def run_scope_command( - *, - request: ScopeCommandRequest, - load_scope_index: ScopeIndexLoader = _native.load_or_build_scope_index, - output_stream: TextIO | None = None, -) -> int: - """Load offline compiler facts, run one pure query, and present its result.""" - - project_dir: Path = request.project_dir if request.project_dir is not None else Path.cwd() - index: ScopeIndex = load_scope_index(project_dir=project_dir, no_cache=request.no_cache) - lookup: ScopeLookup = build_scope_lookup(index=index) - request = replace(request, target=qualify_scope_target(lookup=lookup, target=request.target)) - filters: ScopeReportFilters = ScopeReportFilters( - include_nearby=request.include_nearby, - defined_under=request.defined_under, - kinds=tuple(DeclarationKind(kind) for kind in request.kinds), - glob=request.match, - used_only=request.used_only, - dependency_depth=request.dependency_depth, - cursor=request.after, - page_size=request.page_size, - nearby_depth=request.nearby_depth, - globals=request.globals, - ) - result: ScopeReport | ScopeBrowseResult | ScopeListResult - if request.browse is not None: - result = browse_scope_folders( - lookup=lookup, - folder=request.browse, - target=request.target, - target_is_prospective=request.at is not None, - ) - elif request.list_path is not None: - result = list_scope_declarations( - lookup=lookup, - folder=request.list_path, - filters=filters, - target=request.target, - target_is_prospective=request.at is not None, - ) - else: - directory: bool = request.at is not None and request.at.endswith(("/", "\\")) - report: ScopeReport = query_scope_report( - lookup=lookup, - target=request.target, - at=request.at, - directory=directory, - filters=filters, - ) - if request.explain is not None: - explanation, diagnostics = explain_scope_declaration( - lookup=lookup, - declaration=request.explain, - target=request.target, - at=request.at, - directory=directory, - ) - report = replace( - report, - explanation=explanation, - diagnostics=(*report.diagnostics, *diagnostics), - complete=report.complete and explanation.complete, - ) - if request.as_path is not None: - move, diagnostics = preview_scope_move( - lookup=lookup, resource=request.target or "", destination=request.as_path - ) - report = replace( - report, - move_preview=move, - diagnostics=(*report.diagnostics, *diagnostics), - complete=report.complete and move is not None and move.complete, - ) - result = report - stream: TextIO = output_stream if output_stream is not None else sys.stdout - output: str = ( - serialize_scope_report(report=result) - if request.json_output - else render_scope_result( - result=result, - request=request, - use_color=output_stream is None and not request.no_color and supports_color(), - ) - ) - stream.write(output) - return 1 if _failed(result=result) else 0 - - -def _failed(*, result: ScopeReport | ScopeBrowseResult | ScopeListResult) -> bool: - if isinstance(result, ScopeReport): - complete: bool = result.complete - diagnostics: tuple[ScopeDiagnostic, ...] = result.diagnostics - elif isinstance(result, ScopeBrowseResult): - complete = result.complete - diagnostics = result.diagnostics - else: - complete = result.section.complete - diagnostics = result.diagnostics - return not complete or any(item.severity is DiagnosticSeverity.ERROR for item in diagnostics) diff --git a/src/sqlbuild/cli/commands/_helpers/scope/output.py b/src/sqlbuild/cli/commands/_helpers/scope/output.py deleted file mode 100644 index b55529ffa6..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scope/output.py +++ /dev/null @@ -1,561 +0,0 @@ -"""Deterministic, value-safe text presentation for compiler scope results.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import PurePosixPath -from shlex import quote - -from sqlbuild.cli.commands.constants import ( - SCOPE_DEFAULT_PAGE_SIZE, - SCOPE_GLOBAL_ALL, - SCOPE_GLOBAL_SUMMARY, - SCOPE_PATH_COMPACT, - SCOPE_PATH_NONE, - SCOPE_PATH_RELATIVE, -) -from sqlbuild.cli.commands.models import ScopeCommandRequest -from sqlbuild.compiler.scopes.models import ( - DeclarationReport, - MovePreview, - ScopeBrowseResult, - ScopeDiagnostic, - ScopeListResult, - ScopeReport, - ScopeResourceReport, - ScopeSection, - SourceLocation, -) -from sqlbuild.compiler.scopes.types import DiagnosticSeverity -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.count_noun import format_count_noun - -_USED_MARKER: str = "●" -_UNUSED_MARKER: str = "○" -_LEGEND_MARKERS: str = ( - f"{_USED_MARKER} used by this resource {_UNUSED_MARKER} not used by this resource" -) -_LEGEND_VERBOSE_HINT: str = "--verbose for scope details" - - -@dataclass(frozen=True) -class _ScopeText: - """Styling and detail options shared by one scope text rendering.""" - - request: ScopeCommandRequest - style: CliStyle - - @property - def verbose(self) -> bool: - return self.request.verbose - - -def render_scope_result( - *, - result: ScopeReport | ScopeBrowseResult | ScopeListResult, - request: ScopeCommandRequest, - use_color: bool = False, -) -> str: - """Render one compiler-owned scope result without deriving scope facts.""" - - text: _ScopeText = _ScopeText(request=request, style=CliStyle(use_color=use_color)) - if isinstance(result, ScopeBrowseResult): - return _render_browse(result=result, text=text) - if isinstance(result, ScopeListResult): - return _render_list(result=result, text=text) - return _render_report(report=result, text=text) - - -def _render_report(*, report: ScopeReport, text: _ScopeText) -> str: - request: ScopeCommandRequest = text.request - style: CliStyle = text.style - resource: ScopeResourceReport = report.resource - lines: list[str] = [ - style.section("Scope"), - _field(label="Target", value=resource.target, text=text), - ] - if resource.identity is not None: - lines.append(_field(label="Resource", value=resource.identity, text=text)) - if resource.path is not None: - lines.append( - _field(label="Path", value=_path(value=resource.path, mode=request.paths), text=text) - ) - labels: list[str] = [] - if resource.prospective: - labels.append("prospective") - if resource.directory: - labels.append("directory") - if resource.duplicate_count: - labels.append( - format_count_noun(count=resource.duplicate_count, singular="match", plural="matches") - ) - if labels: - lines.append(_field(label="Status", value=", ".join(labels), text=text)) - used: frozenset[str] = frozenset(item.identity for item in report.used) - lines.extend(["", _header(title="Used", count=f"({len(report.used)})", text=text)]) - lines.extend(_declaration_lines(declarations=report.used, text=text, used=used)) - lines.extend(["", style.section("Scope chain")]) - if report.scope_chain: - for index, entry in enumerate(report.scope_chain): - chain_path: str = _path(value=entry.path, mode=request.paths) - lines.append( - f" {_connector(last=index == len(report.scope_chain) - 1, text=text)} " - f"{_scope_label(entry.kind)} " - f"{style.muted(f'{chain_path} ({entry.declaration_count})')}" - ) - else: - lines.append(f" {style.muted('(none)')}") - for title, declarations, section, optional in ( - ("Available", report.available, _section(report=report, name="available"), False), - ( - "Relationship grants", - report.relationship_scope, - _section(report=report, name="relationship_scope"), - True, - ), - ( - "Nearby unavailable", - report.nearby_unavailable, - _section(report=report, name="nearby_unavailable"), - True, - ), - ): - collapsed_count: int = section.collapsed_count if section is not None else 0 - if optional and not text.verbose and not declarations and not collapsed_count: - continue - lines.extend( - [ - "", - _header( - title=title, - count=_section_count(section=section, count=len(declarations)), - text=text, - ), - ] - ) - lines.extend(_declaration_lines(declarations=declarations, text=text, used=used)) - if collapsed_count: - global_count: str = format_count_noun(count=collapsed_count, singular="global") - lines.append( - " " - + style.muted( - f"… {global_count} collapsed; run " - f"{_follow_up(request=request, globals_all=True)}" - ) - ) - if report.explanation is not None: - lines.extend(["", style.section("Explanation")]) - if report.explanation.declaration is None: - lines.append(f" {style.muted('(declaration not resolved)')}") - else: - lines.extend( - _explanation_lines(declaration=report.explanation.declaration, text=text, used=used) - ) - if report.move_preview is not None: - lines.extend(["", *_move_lines(move=report.move_preview, text=text, used=used)]) - if not text.verbose: - lines.extend(["", style.muted(f"{_LEGEND_MARKERS} {_LEGEND_VERBOSE_HINT}")]) - lines.extend( - [ - "", - *_diagnostic_lines(diagnostics=report.diagnostics, text=text), - _completeness_line(complete=report.complete, text=text), - ] - ) - return "\n".join(lines) + "\n" - - -def _render_browse(*, result: ScopeBrowseResult, text: _ScopeText) -> str: - request: ScopeCommandRequest = text.request - style: CliStyle = text.style - lines: list[str] = [ - style.section("Scope folders"), - _field(label="Path", value=_path(value=result.folder, mode=request.paths), text=text), - "", - ] - if not result.folders: - lines.append(f" {style.muted('(none)')}") - for index, folder in enumerate(result.folders): - kinds: str = ( - ", ".join(f"{kind} {count}" for kind, count in folder.kind_counts) or "no declarations" - ) - counts: str = ( - f"{format_count_noun(count=folder.descendant_count, singular='declaration')}, " - f"{folder.used_count} used, " - f"{format_count_noun(count=folder.child_count, singular='child', plural='children')}; " - f"{kinds}" - ) - lines.append( - f" {_connector(last=index == len(result.folders) - 1, text=text)} " - f"{folder.name}/ {style.muted(counts)}" - ) - lines.append(f" {style.muted(_follow_up(request=request, browse=folder.path))}") - lines.append(f" {style.muted(_follow_up(request=request, list_path=folder.path))}") - lines.extend( - [ - "", - *_diagnostic_lines(diagnostics=result.diagnostics, text=text), - _completeness_line(complete=result.complete, text=text), - ] - ) - return "\n".join(lines) + "\n" - - -def _render_list(*, result: ScopeListResult, text: _ScopeText) -> str: - request: ScopeCommandRequest = text.request - style: CliStyle = text.style - section: ScopeSection = result.section - lines: list[str] = [ - style.section("Scope declarations"), - _field(label="Path", value=_path(value=result.folder, mode=request.paths), text=text), - _field(label="Showing", value=f"{section.returned} of {section.total}", text=text), - "", - ] - lines.extend(_declaration_lines(declarations=result.declarations, text=text, used=frozenset())) - if section.next_cursor is not None: - lines.extend( - [ - "", - style.muted("Continue:") - + " " - + _follow_up( - request=request, - list_path=result.folder, - after=section.next_cursor, - ), - ] - ) - if not text.verbose and result.declarations: - lines.extend(["", style.muted(_LEGEND_VERBOSE_HINT)]) - lines.extend( - [ - "", - *_diagnostic_lines(diagnostics=result.diagnostics, text=text), - _completeness_line(complete=section.complete, text=text), - ] - ) - return "\n".join(lines) + "\n" - - -def _declaration_lines( - *, - declarations: tuple[DeclarationReport, ...], - text: _ScopeText, - used: frozenset[str], - used_marker_style: Callable[[str], str] | None = None, - name_style: Callable[[str], str] | None = None, - detailed: bool | None = None, -) -> list[str]: - style: CliStyle = text.style - if not declarations: - return [f" {style.muted('(none)')}"] - show_details: bool = text.verbose if detailed is None else detailed - resolved_used_marker_style: Callable[[str], str] = used_marker_style or style.success - resolved_name_style: Callable[[str], str] = name_style or style.object_name - lines: list[str] = [] - for index, declaration in enumerate(declarations): - marker: str = ( - resolved_used_marker_style(_USED_MARKER) - if declaration.identity in used - else style.muted(_UNUSED_MARKER) - ) - line: str = ( - f" {_connector(last=index == len(declarations) - 1, text=text)} {marker} " - f"{resolved_name_style(declaration.identity)}" - ) - if show_details: - line += f" {style.muted(f'[{"; ".join(_declaration_details(declaration))}]')}" - if text.request.paths != SCOPE_PATH_NONE: - location: SourceLocation = declaration.definition - position: str = ( - f"{location.line}:{location.column}" if show_details else f"{location.line}" - ) - line += " " + style.muted( - f"{_path(value=location.path, mode=text.request.paths)}:{position}" - ) - lines.append(line) - return lines - - -def _declaration_details(declaration: DeclarationReport) -> list[str]: - details: list[str] = [declaration.kind, _scope_label(declaration.scope)] - if declaration.visibility is not None: - provenance: str = declaration.visibility.reason - if declaration.visibility.through is not None: - provenance += f" through {declaration.visibility.through}" - details.append(provenance) - if declaration.inaccessible_reason is not None: - details.append(declaration.inaccessible_reason) - details.extend(_metadata_parts(declaration)) - return details - - -def _scope_label(scope: str) -> str: - return { - "global": "project", - "inherited": "descendant-public", - "local": "exact-owner-private", - "private": "model-private", - }.get(scope, scope) - - -def _metadata_parts(declaration: DeclarationReport) -> list[str]: - metadata: dict[str, object] = dict(declaration.metadata) - parts: list[str] = [] - parameters: object = metadata.get("parameters") - if isinstance(parameters, tuple | list): - parts.append(f"params {len(parameters)}") - members: object = metadata.get("member_count") - if isinstance(members, int): - parts.append(f"members {members}") - scalar_type: object = metadata.get("scalar_type") - if isinstance(scalar_type, str): - parts.append(f"type {scalar_type}") - logical_type: object = metadata.get("logical_type") - if isinstance(logical_type, str): - parts.append(f"type {logical_type}") - collection: object = metadata.get("collection_kind") - if isinstance(collection, str): - item_count: object = metadata.get("item_count") - parts.append(f"{collection} {item_count}" if isinstance(item_count, int) else collection) - role_root: object = metadata.get("role_root") - if isinstance(role_root, str): - parts.append(f"role {role_root}") - bucket_path: object = metadata.get("bucket_path") - if isinstance(bucket_path, str): - parts.append(f"bucket {bucket_path} (navigation only)") - return parts - - -def _explanation_lines( - *, declaration: DeclarationReport, text: _ScopeText, used: frozenset[str] -) -> list[str]: - style: CliStyle = text.style - lines: list[str] = _declaration_lines( - declarations=(declaration,), text=text, used=used, detailed=True - ) - required_scope: str | None = ( - _scope_label(declaration.required_scope) if declaration.required_scope is not None else None - ) - facts: tuple[tuple[str, object], ...] = ( - ("Owner", declaration.owner), - ("Owning path", declaration.owning_path), - ("Consumers", declaration.consumers), - ("Dependencies", declaration.dependencies), - ("Grants", declaration.grants), - ("Required scope", required_scope), - ("Required path", declaration.required_path), - ("Promotion impact", declaration.promotion_impact), - ) - for label, value in facts: - rendered: str - if isinstance(value, tuple): - rendered = ", ".join(value) if value else style.muted("(none)") - else: - rendered = str(value) if value is not None else style.muted("(none)") - lines.append(f" {style.muted(f'{label}:')} {rendered}") - return lines - - -def _move_lines(*, move: MovePreview, text: _ScopeText, used: frozenset[str]) -> list[str]: - request: ScopeCommandRequest = text.request - style: CliStyle = text.style - lines: list[str] = [ - style.section("Move preview"), - _field(label="Resource", value=move.resource, text=text), - _field( - label="Destination", value=_path(value=move.destination, mode=request.paths), text=text - ), - _field( - label="Ownership root", - value=_path(value=move.new_ownership_root, mode=request.paths), - text=text, - ), - ] - lost_warning: bool = bool(move.lost) - sections: tuple[ - tuple[ - str, - tuple[DeclarationReport, ...], - bool, - Callable[[str], str] | None, - Callable[[str], str] | None, - Callable[[str], str] | None, - ], - ..., - ] = ( - ("Retained", move.retained, False, None, None, None), - ("Gained", move.gained, False, None, None, style.success), - ( - "Lost", - move.lost, - False, - style.warning_strong if lost_warning else None, - style.error, - None, - ), - ("Private retained", move.private_retained, True, None, None, None), - ("Relationship retained", move.relationship_retained, True, None, None, None), - ) - for title, values, optional, title_style, used_marker_style, name_style in sections: - if optional and not text.verbose and not values: - continue - header: str = _header( - title=title, - count=f"({len(values)})", - text=text, - title_style=title_style, - count_style=style.warning if title_style is not None else None, - ) - lines.append(f" {header}") - lines.extend( - " " + line - for line in _declaration_lines( - declarations=values, - text=text, - used=used, - used_marker_style=used_marker_style, - name_style=name_style, - ) - ) - invalidated: tuple[str, ...] = move.invalidated_usages - invalidated_header: str = _header( - title="Invalidated usages", - count=f"({len(invalidated)})", - text=text, - title_style=style.error_strong if invalidated else None, - count_style=style.error if invalidated else None, - ) - lines.append(f" {invalidated_header}") - lines.extend(f" {style.error(f'- {identity}')}" for identity in invalidated) - if not invalidated: - lines.append(f" {style.muted('(none)')}") - return lines - - -def _diagnostic_lines(*, diagnostics: tuple[ScopeDiagnostic, ...], text: _ScopeText) -> list[str]: - style: CliStyle = text.style - if not diagnostics and not text.verbose: - return [] - lines: list[str] = [ - _header( - title="Diagnostics", - count=f"({len(diagnostics)})", - text=text, - title_style=style.error_strong if diagnostics else None, - count_style=style.error if diagnostics else None, - ) - ] - if not diagnostics: - lines.append(f" {style.muted('(none)')}") - for diagnostic in diagnostics: - severity: str = diagnostic.severity.value.upper() - severity_text: str = ( - style.error_strong(severity) - if diagnostic.severity is DiagnosticSeverity.ERROR - else style.warning_strong(severity) - ) - location: str = f" {style.muted(diagnostic.path)}" if diagnostic.path is not None else "" - lines.append(f" {severity_text} {diagnostic.code.value}{location}: {diagnostic.message}") - return lines - - -def _section(*, report: ScopeReport, name: str) -> ScopeSection | None: - return next((section for section in report.sections if section.name == name), None) - - -def _section_count(*, section: ScopeSection | None, count: int) -> str: - if section is None: - return f"({count})" - suffix: str = f", {section.collapsed_count} collapsed" if section.collapsed_count else "" - return f"({section.returned} of {section.total}{suffix})" - - -def _header( - *, - title: str, - count: str, - text: _ScopeText, - title_style: Callable[[str], str] | None = None, - count_style: Callable[[str], str] | None = None, -) -> str: - style: CliStyle = text.style - return f"{(title_style or style.section)(title)} {(count_style or style.muted)(count)}" - - -def _field(*, label: str, value: str, text: _ScopeText) -> str: - return f" {text.style.muted(f'{label}:')} {value}" - - -def _connector(*, last: bool, text: _ScopeText) -> str: - return text.style.muted("└─" if last else "├─") - - -def _completeness_line(*, complete: bool, text: _ScopeText) -> str: - style: CliStyle = text.style - state: str = style.success("complete") if complete else style.warning("partial") - return f"{style.muted('Completeness:')} {state}" - - -def _path(*, value: str | None, mode: str) -> str: - if value is None: - return "(none)" - if mode == SCOPE_PATH_NONE: - return "(hidden)" - if mode == SCOPE_PATH_COMPACT: - return PurePosixPath(value).name - return value - - -def _follow_up( - *, - request: ScopeCommandRequest, - browse: str | None = None, - list_path: str | None = None, - after: str | None = None, - globals_all: bool = False, -) -> str: - args: list[str] = ["sqb", "scope"] - if request.target is not None: - args.append(request.target) - else: - args.extend(("--at", request.at or "")) - if browse is not None: - args.extend(("--browse", browse)) - if list_path is not None: - args.extend(("--list", list_path)) - if request.defined_under is not None: - args.extend(("--defined-under", request.defined_under)) - for kind in request.kinds: - args.extend(("--kind", kind)) - if request.match is not None: - args.extend(("--match", request.match)) - if request.used_only: - args.append("--used-only") - if globals_all: - if request.as_path is not None: - args.extend(("--as-path", request.as_path)) - if request.include_nearby: - args.append("--include-nearby") - if request.nearby_depth != 1: - args.extend(("--nearby-depth", str(request.nearby_depth))) - if request.dependency_depth: - args.extend(("--dependency-depth", str(request.dependency_depth))) - if request.explain is not None: - args.extend(("--explain", request.explain)) - if request.page_size != SCOPE_DEFAULT_PAGE_SIZE: - args.extend(("--page-size", str(request.page_size))) - if after is not None: - args.extend(("--after", after)) - if globals_all: - args.extend(("--globals", SCOPE_GLOBAL_ALL)) - elif request.globals != SCOPE_GLOBAL_SUMMARY: - args.extend(("--globals", request.globals)) - if request.paths != SCOPE_PATH_RELATIVE: - args.extend(("--paths", request.paths)) - if request.verbose: - args.append("--verbose") - if request.no_cache: - args.append("--no-cache") - return " ".join(quote(item) for item in args) diff --git a/src/sqlbuild/cli/commands/_helpers/scope/target.py b/src/sqlbuild/cli/commands/_helpers/scope/target.py deleted file mode 100644 index 502a77fb5f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scope/target.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Read a bare `sqb scope` target name the way `sqb lineage` and `sqb rename` read targets.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.lineage.selection import resolve_lineage_target -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import LineageTarget -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.scopes.models import ResourceIdentity, ScopeLookup -from sqlbuild.compiler.scopes.types import ResourceKind - -_KIND_SEPARATOR: str = ":" -_PATH_MARKERS: tuple[str, ...] = ("/", "\\", ".") -_BARE_RESOURCE_TYPES: dict[ResourceKind, CompiledResourceType] = { - ResourceKind.MODEL: CompiledResourceType.MODEL, - ResourceKind.SOURCE: CompiledResourceType.SOURCE, - ResourceKind.SEED: CompiledResourceType.SEED, - ResourceKind.FUNCTION: CompiledResourceType.UDF, -} -_BARE_USAGE: str = ( - "bare names work for models, sources, seeds and functions; write macros, enums, constants, " - "tests and audits with their prefix, for example macro:" -) - - -def qualify_scope_target(*, lookup: ScopeLookup, target: str | None) -> str | None: - """Return `target` with its kind prefix when it is a bare model, source, seed or function.""" - - if target is None or _KIND_SEPARATOR in target or any(m in target for m in _PATH_MARKERS): - return target - bare_matches: dict[str, ResourceIdentity] = {} - identity: ResourceIdentity - for identity in lookup.resources: - if identity.kind in _BARE_RESOURCE_TYPES and identity.name == target: - bare_matches[str(identity.kind)] = identity - all_keys: dict[str, CompiledObjectKey] = { - identity.name: CompiledObjectKey( - resource_type=_BARE_RESOURCE_TYPES[identity.kind], name=identity.name - ) - for identity in bare_matches.values() - } - resolved: LineageTarget = resolve_lineage_target(all_keys=all_keys, target=target) - prefixed_matches: tuple[str, ...] = _prefix_required_matches(lookup=lookup, name=target) - if resolved.key is None: - if prefixed_matches: - raise CliUserError( - f"scope target '{target}' needs its prefix: {_or_list(prefixed_matches)}", - code="C960", - help=_BARE_USAGE, - ) - raise CliUserError(f"unknown scope target '{target}'", code="C960", help=_BARE_USAGE) - candidates: tuple[str, ...] = ( - *(f"{kind}:{target}" for kind in sorted(bare_matches)), - *prefixed_matches, - ) - if len(candidates) > 1: - raise CliUserError( - f"scope target '{target}' matches more than one resource or declaration; " - f"write {_or_list(candidates)}", - code="C960", - help=_BARE_USAGE, - ) - return candidates[0] - - -def _prefix_required_matches(*, lookup: ScopeLookup, name: str) -> tuple[str, ...]: - resources: set[str] = { - f"{identity.kind}:{name}" - for identity in lookup.resources - if identity.kind not in _BARE_RESOURCE_TYPES and identity.name == name - } - declarations: set[str] = { - f"{identity.kind}:{name}" - for identity in lookup.declarations - if identity.owner is None and identity.name == name - } - return tuple(sorted(resources | declarations)) - - -def _or_list(values: tuple[str, ...]) -> str: - if len(values) == 1: - return values[0] - return ", ".join(values[:-1]) + f" or {values[-1]}" diff --git a/src/sqlbuild/cli/commands/_helpers/seed/execution.py b/src/sqlbuild/cli/commands/_helpers/seed/execution.py index 3806f4d4d2..16b4993020 100644 --- a/src/sqlbuild/cli/commands/_helpers/seed/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/seed/execution.py @@ -7,17 +7,17 @@ from dataclasses import replace from typing import TextIO +from sqlbuild._native import ( + ConnectionProgressReporter, + ExecutionEventWriter, + NativeProgressProjector, + current_native_progress_projector, +) from sqlbuild.cli.commands.models import ( SeedExecutionPreparation, SeedInvocation, SeedRunOutcome, ) -from sqlbuild.cli.output.classes.execution_event_writer import ExecutionEventWriter -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter -from sqlbuild.cli.progress.classes.native_progress_projector import ( - NativeProgressProjector, - current_native_progress_projector, -) from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.build.types import ExecutionStatus diff --git a/src/sqlbuild/cli/commands/_helpers/seed/outputs.py b/src/sqlbuild/cli/commands/_helpers/seed/outputs.py index 01a1fe7d42..569630a729 100644 --- a/src/sqlbuild/cli/commands/_helpers/seed/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/seed/outputs.py @@ -2,19 +2,22 @@ from __future__ import annotations -from sqlbuild._native import format_seed_execution_json, write_execution_json_output +from sqlbuild._native import ( + format_completion_line, + format_seed_execution_json, + format_summary_footer, + write_execution_header, + write_execution_json_output, +) from sqlbuild.cli.commands.models import ( SeedCommandRequest, SeedExecutionPreparation, SeedInvocation, SeedRunOutcome, ) -from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.build.types import ExecutionStatus from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.completion_line import format_completion_line -from sqlbuild.presentation.main.summary_footer import format_summary_footer from sqlbuild.presentation.main.surface_header import format_surface_header from sqlbuild.presentation.main.tree_connector import tree_connector from sqlbuild.presentation.types import CompletionState diff --git a/src/sqlbuild/cli/commands/_helpers/seed/planning.py b/src/sqlbuild/cli/commands/_helpers/seed/planning.py index 95632c8c8a..e91cd09bb6 100644 --- a/src/sqlbuild/cli/commands/_helpers/seed/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/seed/planning.py @@ -4,7 +4,7 @@ from typing import Any -from sqlbuild._native import build_static_pipeline_result +from sqlbuild._native import ConnectionProgressReporter, build_static_pipeline_result from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) @@ -13,7 +13,6 @@ SeedExecutionPreparation, SeedInvocation, ) -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter from sqlbuild.compiler.compile.types import CompiledSeedView from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context diff --git a/src/sqlbuild/cli/commands/_helpers/skills/__init__.py b/src/sqlbuild/cli/commands/_helpers/skills/__init__.py deleted file mode 100644 index 410f671443..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/skills/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Skills command helpers.""" diff --git a/src/sqlbuild/cli/commands/_helpers/skills/update.py b/src/sqlbuild/cli/commands/_helpers/skills/update.py deleted file mode 100644 index ac325980a7..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/skills/update.py +++ /dev/null @@ -1,391 +0,0 @@ -"""Install SQLBuild skill files into agent-specific locations.""" - -from __future__ import annotations - -import tomllib -from importlib.resources import files -from importlib.resources.abc import Traversable -from os import replace -from pathlib import Path -from tempfile import NamedTemporaryFile - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.output.models import ( - SkillInstallTarget, - SkillMaintenanceResult, - SkillSettings, - SkillUpdateResult, -) -from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME - -generated_marker: str = "" -legacy_generated_marker: str = "" -generated_markers: tuple[str, ...] = (generated_marker, legacy_generated_marker) -skill_name: str = "sqlbuild" -skill_source_package: str = "sqlbuild" -skill_source_dir: str = ".agents/skills/sqlbuild" -skill_entry_file: str = "SKILL.md" -skill_file_suffix: str = ".md" -valid_skill_targets: tuple[str, ...] = ("opencode", "claude", "agents") -default_skill_targets: tuple[str, ...] = ("agents", "claude") -opencode_skill_target: str = "opencode" -claude_skill_target: str = "claude" - - -def update_sqlbuild_skills( - *, - project_dir: Path, - global_install: bool = False, - requested_targets: tuple[str, ...] = (), - force: bool = False, - home_dir: Path | None = None, -) -> SkillUpdateResult: - """Write packaged SQLBuild skill content to configured agent locations.""" - - target_names: tuple[str, ...] = resolve_skill_target_names( - project_dir=project_dir, - requested_targets=requested_targets, - ) - install_targets: tuple[SkillInstallTarget, ...] = build_install_targets( - project_dir=project_dir, - target_names=target_names, - global_install=global_install, - home_dir=home_dir, - ) - expected_files: dict[str, str] = load_packaged_skill_files() - if not force: - collisions: list[Path] = [] - for install_target in install_targets: - collisions.extend( - _skill_collisions( - skill_dir=install_target.path.parent, expected_files=expected_files - ) - ) - if collisions: - raise CliUserError( - f"refusing to overwrite non-generated skill file: {collisions[0]}", - code="C806", - help="rerun with --force to replace it", - ) - written_paths: list[Path] = [] - for install_target in install_targets: - _install_skill_directory( - skill_dir=install_target.path.parent, expected_files=expected_files, force=force - ) - written_paths.append(install_target.path) - - return SkillUpdateResult(written_paths=tuple(written_paths)) - - -def resolve_skill_target_names( - *, project_dir: Path, requested_targets: tuple[str, ...] = () -) -> tuple[str, ...]: - if requested_targets: - return normalize_skill_targets(targets=requested_targets, source="--target") - - settings: SkillSettings = load_project_skill_settings(project_dir=project_dir) - if settings.targets: - return settings.targets - - return default_skill_targets - - -def normalize_skill_targets(*, targets: tuple[str, ...], source: str) -> tuple[str, ...]: - normalized_targets: list[str] = [] - seen_targets: set[str] = set() - for target in targets: - normalized_target: str = target.strip().lower() - if normalized_target not in valid_skill_targets: - allowed_targets: str = ", ".join(valid_skill_targets) - raise CliUserError( - f"{source} contains unsupported skill target '{target}'", - code="C801", - help=f"supported targets are: {allowed_targets}", - ) - if normalized_target in seen_targets: - continue - normalized_targets.append(normalized_target) - seen_targets.add(normalized_target) - - return tuple(normalized_targets) - - -def load_project_skill_settings(*, project_dir: Path) -> SkillSettings: - config_path: Path = project_dir / PROJECT_CONFIG_FILENAME - if not config_path.exists(): - return SkillSettings(targets=(), auto_update=False, configured=False) - - try: - payload: object = tomllib.loads(config_path.read_text(encoding="utf-8-sig")) - except tomllib.TOMLDecodeError as error: - raise CliUserError( - f"{config_path} contains invalid TOML: {error}", - code="C802", - ) from error - if not isinstance(payload, dict): - return SkillSettings(targets=(), auto_update=False, configured=False) - - skills_payload: object = payload.get("skills") - if skills_payload is None: - return SkillSettings(targets=(), auto_update=False, configured=False) - if not isinstance(skills_payload, dict): - raise CliUserError( - f"{config_path} [skills] must be a table", - code="C803", - ) - - targets_payload: object = skills_payload.get("targets") - if targets_payload is not None and ( - not isinstance(targets_payload, list) - or not all(isinstance(target, str) for target in targets_payload) - ): - raise CliUserError( - f"{config_path} skills.targets must be a list of strings", - code="C804", - ) - auto_update: object = skills_payload.get("auto_update", False) - if not isinstance(auto_update, bool): - raise CliUserError( - f"{config_path} skills.auto_update must be a boolean", - code="C808", - ) - targets: tuple[str, ...] = ( - () - if targets_payload is None - else normalize_skill_targets(targets=tuple(targets_payload), source="skills.targets") - ) - return SkillSettings( - targets=targets or default_skill_targets, - auto_update=auto_update, - configured=True, - ) - - -def maintain_sqlbuild_skills(*, project_dir: Path) -> SkillMaintenanceResult: - """Check or refresh generated project skills without changing command success.""" - - settings: SkillSettings = load_project_skill_settings(project_dir=project_dir) - target_names: tuple[str, ...] = settings.targets - if not settings.configured: - target_names = tuple( - target_name - for target_name in valid_skill_targets - if _local_install_path(project_dir=project_dir, target_name=target_name).exists() - ) - if not target_names: - return SkillMaintenanceResult() - - install_targets: tuple[SkillInstallTarget, ...] = build_install_targets( - project_dir=project_dir, - target_names=target_names, - ) - expected_files: dict[str, str] = load_packaged_skill_files() - stale_paths: list[Path] = [] - collision_paths: list[Path] = [] - for install_target in install_targets: - skill_dir: Path = install_target.path.parent - collisions: tuple[Path, ...] = _skill_collisions( - skill_dir=skill_dir, expected_files=expected_files - ) - if collisions: - collision_paths.extend(collisions) - continue - if _skill_directory_is_stale(skill_dir=skill_dir, expected_files=expected_files): - stale_paths.append(install_target.path) - - if settings.auto_update and stale_paths: - for path in stale_paths: - _install_skill_directory(skill_dir=path.parent, expected_files=expected_files) - message: str = "Updated stale SQLBuild skill files:\n" + "".join( - f" {path}\n" for path in stale_paths - ) - if collision_paths: - message += _stale_skill_message(collision_paths=collision_paths) - return SkillMaintenanceResult(message=message) - if stale_paths or collision_paths: - return SkillMaintenanceResult(message=_stale_skill_message(collision_paths=collision_paths)) - return SkillMaintenanceResult() - - -def _stale_skill_message(*, collision_paths: list[Path]) -> str: - collision_detail: str = "" - if collision_paths: - collision_detail = " Custom files were not overwritten:\n" + "".join( - f" {path}\n" for path in collision_paths - ) - return f"SQLBuild skill files are out of date\n{collision_detail} Run: sqb skills\n" - - -def _local_install_path(*, project_dir: Path, target_name: str) -> Path: - return build_install_targets( - project_dir=project_dir, - target_names=(target_name,), - )[0].path - - -def build_install_targets( - *, - project_dir: Path, - target_names: tuple[str, ...], - global_install: bool = False, - home_dir: Path | None = None, -) -> tuple[SkillInstallTarget, ...]: - home_path: Path = home_dir if home_dir is not None else Path.home() - local_install_root: Path = _find_git_root(project_dir=project_dir) or project_dir - install_targets: list[SkillInstallTarget] = [] - for target_name in target_names: - base_path: Path - if target_name == opencode_skill_target: - base_path = ( - home_path / ".config/opencode" - if global_install - else local_install_root / ".opencode" - ) - elif target_name == claude_skill_target: - base_path = home_path / ".claude" if global_install else local_install_root / ".claude" - else: - base_path = home_path / ".agents" if global_install else local_install_root / ".agents" - install_targets.append( - SkillInstallTarget( - name=target_name, - path=base_path / "skills" / skill_name / "SKILL.md", - ) - ) - return tuple(install_targets) - - -def _find_git_root(*, project_dir: Path) -> Path | None: - for candidate in (project_dir, *project_dir.parents): - marker: Path = candidate / ".git" - if marker.is_dir() or marker.is_file(): - return candidate - return None - - -def load_packaged_skill_content() -> str: - skill_file: Traversable = _packaged_skill_root().joinpath(skill_entry_file) - if not skill_file.is_file(): - raise CliUserError( - "packaged SQLBuild skill is missing", - code="C805", - help="reinstall SQLBuild or report a packaging issue", - ) - return skill_file.read_text(encoding="utf-8") - - -def ensure_generated_marker(content: str) -> str: - if generated_marker in content: - return content - if legacy_generated_marker in content: - return content.replace(legacy_generated_marker, generated_marker, 1) - if content.startswith("---\n"): - frontmatter_end: int = content.find("\n---\n", 4) - if frontmatter_end != -1: - insert_at: int = frontmatter_end + len("\n---\n") - return f"{content[:insert_at]}\n{generated_marker}\n{content[insert_at:]}" - return f"{generated_marker}\n{content}" - - -def write_skill_file(*, path: Path, content: str, force: bool = False) -> None: - if path.exists(): - existing_content: str = path.read_text(encoding="utf-8") - if not any(marker in existing_content for marker in generated_markers) and not force: - raise CliUserError( - f"refusing to overwrite non-generated skill file: {path}", - code="C806", - help="rerun with --force to replace it", - ) - - path.parent.mkdir(parents=True, exist_ok=True) - with NamedTemporaryFile( - mode="w", - encoding="utf-8", - dir=path.parent, - prefix=f".{path.name}.", - delete=False, - ) as temporary_file: - _ = temporary_file.write(content) - temporary_path: Path = Path(temporary_file.name) - replace(temporary_path, path) - - -def load_packaged_skill_files() -> dict[str, str]: - """Return every packaged skill file keyed by skill-relative path, with generated markers.""" - - entry_content: str = load_packaged_skill_content() - collected: dict[str, str] = _collect_skill_files(node=_packaged_skill_root(), prefix="") - collected[skill_entry_file] = entry_content - return { - relative_path: ensure_generated_marker(content) - for relative_path, content in sorted(collected.items()) - } - - -def _packaged_skill_root() -> Traversable: - return files(skill_source_package).joinpath(*skill_source_dir.split("/")) - - -def _collect_skill_files(*, node: Traversable, prefix: str) -> dict[str, str]: - collected: dict[str, str] = {} - for child in node.iterdir(): - if child.name.startswith((".", "__")): - continue - if child.is_dir(): - collected.update(_collect_skill_files(node=child, prefix=f"{prefix}{child.name}/")) - elif child.name.endswith(skill_file_suffix): - collected[f"{prefix}{child.name}"] = child.read_text(encoding="utf-8") - return collected - - -def _is_generated(path: Path) -> bool: - content: str = path.read_text(encoding="utf-8") - return any(marker in content for marker in generated_markers) - - -def _skill_collisions(*, skill_dir: Path, expected_files: dict[str, str]) -> tuple[Path, ...]: - """Return expected install paths occupied by files SQLBuild did not generate.""" - - return tuple( - skill_dir / relative_path - for relative_path in expected_files - if (skill_dir / relative_path).is_file() and not _is_generated(skill_dir / relative_path) - ) - - -def _stale_generated_files(*, skill_dir: Path, expected_files: dict[str, str]) -> tuple[Path, ...]: - if not skill_dir.is_dir(): - return () - return tuple( - path - for path in sorted(skill_dir.rglob(f"*{skill_file_suffix}")) - if path.relative_to(skill_dir).as_posix() not in expected_files and _is_generated(path) - ) - - -def _skill_directory_is_stale(*, skill_dir: Path, expected_files: dict[str, str]) -> bool: - for relative_path, content in expected_files.items(): - path: Path = skill_dir / relative_path - if not path.is_file() or path.read_text(encoding="utf-8") != content: - return True - return bool(_stale_generated_files(skill_dir=skill_dir, expected_files=expected_files)) - - -def _install_skill_directory( - *, skill_dir: Path, expected_files: dict[str, str], force: bool = False -) -> None: - """Write all packaged skill files and remove generated files no longer shipped.""" - - for relative_path, content in expected_files.items(): - path: Path = skill_dir / relative_path - if path.is_file() and path.read_text(encoding="utf-8") == content: - continue - write_skill_file(path=path, content=content, force=force) - for stale_path in _stale_generated_files(skill_dir=skill_dir, expected_files=expected_files): - stale_path.unlink() - for directory in sorted( - (path for path in skill_dir.rglob("*") if path.is_dir()), - key=lambda path: len(path.parts), - reverse=True, - ): - if not any(directory.iterdir()): - directory.rmdir() diff --git a/src/sqlbuild/cli/commands/_helpers/test/execution.py b/src/sqlbuild/cli/commands/_helpers/test/execution.py index 4c76dcb831..8e5d31fd76 100644 --- a/src/sqlbuild/cli/commands/_helpers/test/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/test/execution.py @@ -4,6 +4,12 @@ from collections.abc import Callable +from sqlbuild._native import ( + ConnectionProgressReporter, + ExecutionEventWriter, + NativeProgressProjector, + current_native_progress_projector, +) from sqlbuild.cli.commands._helpers.test.sql_progress import ( build_test_expectation_rows, format_parameterized_test_label, @@ -15,14 +21,6 @@ TestExecutionPreparation, TestInvocation, ) -from sqlbuild.cli.output.classes.execution_event_writer import ExecutionEventWriter -from sqlbuild.cli.progress.classes.connection_progress_reporter import ( - ConnectionProgressReporter, -) -from sqlbuild.cli.progress.classes.native_progress_projector import ( - NativeProgressProjector, - current_native_progress_projector, -) from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( NestedCommandProgressCallbacks, ) diff --git a/src/sqlbuild/cli/commands/_helpers/test/outputs.py b/src/sqlbuild/cli/commands/_helpers/test/outputs.py index fffb9fb452..dca9332ef6 100644 --- a/src/sqlbuild/cli/commands/_helpers/test/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/test/outputs.py @@ -2,14 +2,17 @@ from __future__ import annotations -from sqlbuild._native import format_test_execution_json, write_execution_json_output +from sqlbuild._native import ( + format_summary_footer, + format_test_execution_json, + write_execution_json_output, +) from sqlbuild.cli.commands._helpers.test.sql_progress import format_parameterized_test_label from sqlbuild.cli.commands.models import TestCommandRequest, TestInvocation from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.main.coded_error_text import format_coded_error -from sqlbuild.presentation.main.summary_footer import format_summary_footer _ERROR_LABEL_WIDTH: int = 10 diff --git a/src/sqlbuild/cli/commands/_helpers/test/sql_progress.py b/src/sqlbuild/cli/commands/_helpers/test/sql_progress.py index ca177d6577..ffce24acfb 100644 --- a/src/sqlbuild/cli/commands/_helpers/test/sql_progress.py +++ b/src/sqlbuild/cli/commands/_helpers/test/sql_progress.py @@ -5,15 +5,17 @@ import json from pathlib import Path +from sqlbuild._native import ( + format_expectation_detail, + format_expectation_name, + resolve_name_column_width, +) from sqlbuild.cli.progress.constants import NESTED_CHILD_NAME_INDENT -from sqlbuild.cli.progress.main._expectation_detail import format_expectation_detail -from sqlbuild.cli.progress.main._expectation_name import format_expectation_name from sqlbuild.cli.progress.models import NestedProgressChildRow from sqlbuild.compiler.discovery.models import SqlTestParameterDeclaration from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.presentation.main.resolve_name_column_width import resolve_name_column_width from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind diff --git a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py index 58c7d8adda..eb5a92b134 100644 --- a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py +++ b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py @@ -9,16 +9,19 @@ from dataclasses import replace from pathlib import Path -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.commands._helpers.test.sql_progress import format_parameterized_test_label -from sqlbuild.cli.output.classes.execution_event_writer import ExecutionEventWriter -from sqlbuild.cli.progress.classes.native_progress_projector import ( +from sqlbuild._native import ( + ExecutionEventWriter, NativeProgressProjector, current_native_progress_projector, + format_completion_line, + format_expectation_detail, + format_expectation_name, + format_status_cell, + format_summary_footer, ) -from sqlbuild.cli.progress.main._expectation_detail import format_expectation_detail -from sqlbuild.cli.progress.main._expectation_name import format_expectation_name +from sqlbuild.adapter.contract.models import LifeCycleEvent +from sqlbuild.adapter.contract.types import LifeCycleEventKind +from sqlbuild.cli.commands._helpers.test.sql_progress import format_parameterized_test_label from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.types import CompiledResourceType @@ -56,10 +59,7 @@ from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator from sqlbuild.presentation.main.coded_error_text import format_coded_error -from sqlbuild.presentation.main.completion_line import format_completion_line from sqlbuild.presentation.main.inline_error_lines import format_inline_error_lines -from sqlbuild.presentation.main.status_cell import format_status_cell -from sqlbuild.presentation.main.summary_footer import format_summary_footer from sqlbuild.presentation.main.terminal_columns import terminal_columns from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator from sqlbuild.presentation.main.tree_connector import tree_connector diff --git a/src/sqlbuild/cli/commands/classes/load_progress_reporter.py b/src/sqlbuild/cli/commands/classes/load_progress_reporter.py index 22aecbf6fa..b3ed384d04 100644 --- a/src/sqlbuild/cli/commands/classes/load_progress_reporter.py +++ b/src/sqlbuild/cli/commands/classes/load_progress_reporter.py @@ -5,18 +5,18 @@ import threading from typing import TextIO -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.progress.classes.native_progress_projector import ( +from sqlbuild._native import ( NativeProgressProjector, current_native_progress_projector, + format_completion_line, + format_summary_footer, ) +from sqlbuild.adapter.contract.models import LifeCycleEvent +from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.completion_line import format_completion_line -from sqlbuild.presentation.main.summary_footer import format_summary_footer from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator from sqlbuild.presentation.types import CompletionState from sqlbuild.spec.contracts.models import SourceEntry diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index c3fbff3d6b..22b5406d2c 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -2,13 +2,12 @@ from types import MappingProxyType -from sqlbuild.cli.commands.types import CliCommand, PlaygroundTemplate +from sqlbuild.cli.commands.types import CliCommand from sqlbuild.spec.contracts.types import WarehouseGroup C0_CONTROL_CODE_LIMIT: int = 32 C1_CONTROL_CODE_START: int = 127 C1_CONTROL_CODE_LIMIT: int = 160 -RICH_LINEAGE_STATUS_MODEL_THRESHOLD: int = 100 EMPTY_DAG_PATH: str = "" TARGET_DIRECTORY_NAME: str = "target" SQLBUILD_CONCURRENCY_ENV_VAR: str = "SQLBUILD_CONCURRENCY" @@ -16,13 +15,8 @@ DBT_INIT_COMMAND: str = "init" SCENARIO_TEST_COMMAND: str = "test" SCENARIO_CAPTURE_COMMAND: str = "capture" -UNLIMITED_DEPTH_VALUE: str = "all" -UPSTREAM_DIRECTION: str = "upstream" -DOWNSTREAM_DIRECTION: str = "downstream" -BOTH_DIRECTIONS: str = "both" JSON_OUTPUT_FORMAT: str = "json" CSV_OUTPUT_FORMAT: str = "csv" -LIST_OUTPUT_FORMAT: str = "list" DEBUG_COMMAND: str = "debug" LINEAGE_COMMAND: str = "lineage" QUERY_COMMAND: str = "query" @@ -30,16 +24,6 @@ RULES_SHOW_ACTION: str = "show" RULES_SKILLS_ACTION: str = "skills" SQL_FILE_SUFFIX: str = ".sql" -SELECTOR_INTERSECTION_MARKER: str = "," -COLUMN_TARGET_SEPARATOR: str = "." -COLUMN_TARGET_KIND: str = "column" -PLAYGROUND_ORCHESTRATED_PROJECT_DIR: str = "waffle_shop" -PLAYGROUND_ORCHESTRATED_TEMPLATES: frozenset[PlaygroundTemplate] = frozenset( - {PlaygroundTemplate.DAGSTER, PlaygroundTemplate.RIVERS} -) -PLAYGROUND_TEMPLATE_VALUES: tuple[str, ...] = tuple( - template.value for template in PlaygroundTemplate -) SUCCESS_STATUS: str = "success" FAILED_STATUS: str = "failed" DEFAULT_MAX_SNAPSHOT_ROWS_PER_RELATION: int = 10_000 @@ -159,32 +143,3 @@ CliCommand.SKILLS: None, } ) -LAYER_FOLDER_RULE_CODES: frozenset[str] = frozenset({"SQBRPROJECT102", "SQBRPROJECT105"}) -MODEL_NAME_LAYER_SEPARATOR: str = "__" -MODEL_NAME_LAYER_INDEX: int = 1 -MODEL_NAME_PART_COUNTS: frozenset[int] = frozenset({3, 4}) -LAYER_FOLDERS: MappingProxyType[str, tuple[str, ...]] = MappingProxyType( - { - "stg": ("staging",), - "stg_v": ("staging",), - "int_clean": ("intermediate", "clean"), - "int_enriched": ("intermediate", "enriched"), - "int_v": ("intermediate", "enriched"), - "mart": ("mart",), - "mart_v": ("mart",), - } -) -CONTIGUOUS_LAYER_FOLDERS: MappingProxyType[str, tuple[str, ...]] = MappingProxyType( - { - "stg": ("staging",), - "stg_v": ("staging",), - "int_clean": ("intermediate", "clean"), - "int_enriched": ("intermediate", "enriched"), - "int_v": ("intermediate",), - "mart": ("mart",), - "mart_v": ("mart",), - } -) -WINDOWS_OS_NAME: str = "nt" -FOLDER_PLACEHOLDER: str = "" -PROJECT_DIR_OPTION: str = "--project-dir" diff --git a/src/sqlbuild/cli/commands/main/dbt/_dbt.py b/src/sqlbuild/cli/commands/main/dbt/_dbt.py index 874c431ee6..6e8e39dd0c 100644 --- a/src/sqlbuild/cli/commands/main/dbt/_dbt.py +++ b/src/sqlbuild/cli/commands/main/dbt/_dbt.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import PlanningProgressReporter from sqlbuild.cli.commands._helpers.dbt.auto_init import ( ensure_sqlbuild_project_for_dbt_command, ) @@ -15,7 +16,6 @@ DBT_VERBOSE_OPTIONS, ) from sqlbuild.cli.commands.main.dbt._dbt_debug import run_dbt_debug_command -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.integrations.dbt.main.cli.validate_execution_args import validate_dbt_execution_args from sqlbuild.integrations.dbt.main.pipeline.execute import execute_dbt_interop_from_project from sqlbuild.integrations.dbt.main.pipeline.plan import plan_dbt_interop_from_project diff --git a/src/sqlbuild/cli/commands/main/dbt/_dbt_debug.py b/src/sqlbuild/cli/commands/main/dbt/_dbt_debug.py index cefe3afd3f..7294fc02e8 100644 --- a/src/sqlbuild/cli/commands/main/dbt/_dbt_debug.py +++ b/src/sqlbuild/cli/commands/main/dbt/_dbt_debug.py @@ -5,8 +5,8 @@ import sys from pathlib import Path +from sqlbuild._native import run_debug_command from sqlbuild.cli.commands.constants import DBT_NO_CONNECTION_OPTION -from sqlbuild.cli.commands.main.inspection._debug import run_debug as run_sqlbuild_debug from sqlbuild.integrations.dbt.main.pipeline.debug import debug_dbt_from_project from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.presentation.main.supports_color import supports_color @@ -40,7 +40,7 @@ def run_dbt_debug_command( ) status.start("Running SQLBuild diagnostics...") try: - sqlbuild_returncode: int = run_sqlbuild_debug( + sqlbuild_returncode: int = run_debug_command( project_dir=project_dir, no_color=no_color, no_connection=DBT_NO_CONNECTION_OPTION in args, diff --git a/src/sqlbuild/cli/commands/main/execution/_audit.py b/src/sqlbuild/cli/commands/main/execution/_audit.py index 092923fb58..de3e0401ac 100644 --- a/src/sqlbuild/cli/commands/main/execution/_audit.py +++ b/src/sqlbuild/cli/commands/main/execution/_audit.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import write_execution_header from sqlbuild.cli.commands._helpers.audit.execution import ( execute_audit_plan, prepare_audit_execution, @@ -18,7 +19,6 @@ AuditExecutionPreparation, AuditInvocation, ) -from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/cli/commands/main/execution/_check.py b/src/sqlbuild/cli/commands/main/execution/_check.py index 26fb6b9705..df13f84336 100644 --- a/src/sqlbuild/cli/commands/main/execution/_check.py +++ b/src/sqlbuild/cli/commands/main/execution/_check.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import write_execution_header from sqlbuild.cli.commands._helpers.check.execution import ( execute_check_plan, prepare_check_execution, @@ -17,7 +18,6 @@ CheckExecutionPreparation, CheckInvocation, ) -from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult from sqlbuild.provider.classes.session import ProviderSession diff --git a/src/sqlbuild/cli/commands/main/execution/_freshness.py b/src/sqlbuild/cli/commands/main/execution/_freshness.py deleted file mode 100644 index 92a6d43bdb..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_freshness.py +++ /dev/null @@ -1,155 +0,0 @@ -"""CLI source freshness command entry point.""" - -from __future__ import annotations - -import sys -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - build_project_graph, - discover_project_inputs, - write_execution_json_output, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.freshness.observe import ( - observe_source_freshness_for_command, -) -from sqlbuild.cli.commands._helpers.freshness.output import ( - format_freshness_json, - format_freshness_text, -) -from sqlbuild.cli.commands._helpers.freshness.selection import resolve_freshness_source_names -from sqlbuild.cli.commands._helpers.freshness.state import ( - read_direct_freshness_state_for_command, -) -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - FreshnessCommandRequest, - FreshnessCommandResult, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import SourceEntry - - -def run_freshness(request: FreshnessCommandRequest) -> int: - """Observe source freshness without writing state.""" - - selected_target: str | None = request.selected_target - select: tuple[str, ...] = request.select - cli_vars: dict[str, object] | None = request.cli_vars - compare_state: bool = request.compare_state - if request.fail_on_stale and not compare_state: - raise CliUserError("freshness --fail-on-stale requires --state", code="C238") - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - _ = reject_unit_test_selectors( - select=select, exclude=request.exclude, discovered_inputs=discovered_inputs - ) - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, project_dir=effective_project_dir - ) - graph: ProjectGraph = build_project_graph( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=selected_target, - no_sql_validation=request.no_sql_validation, - cli_vars=cli_vars, - ) - sources: tuple[SourceEntry, ...] = tuple( - source.source_entry for source in graph.project.sources - ) - selected_source_names: tuple[str, ...] = resolve_freshness_source_names( - graph=graph, - select=select, - exclude=request.exclude, - ) - if not selected_source_names: - result: FreshnessCommandResult = FreshnessCommandResult() - else: - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - selected_target=selected_target, - cli_vars=cli_vars, - ) - connection: Any = adapter.connect(connection_config) - try: - previous_records_by_source_name: dict[str, SourceFreshnessRecord] | None = None - previous_records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] | None = None - if compare_state: - previous_records = read_direct_freshness_state_for_command( - adapter=adapter, - connection=connection, - project=graph.project, - ) - result = observe_source_freshness_for_command( - adapter=adapter, - connection=connection, - sources=sources, - select=selected_source_names, - exclude=(), - observed_at=_source_freshness_observed_at(), - run_id=graph.project.run_id, - previous_records=previous_records, - previous_records_by_source_name=previous_records_by_source_name, - ) - finally: - adapter.close(connection) - - payload: str = format_freshness_json(result) - if request.json_output: - write_execution_json_output( - payload=payload + "\n", - json_output=True, - json_output_path=request.json_output_path, - ) - else: - if request.json_output_path is not None: - write_execution_json_output( - payload=payload + "\n", - json_output=False, - json_output_path=request.json_output_path, - ) - sys.stdout.write( - _format_freshness_for_terminal( - result=result, - no_color=request.no_color, - ) - ) - if request.fail_on_stale and ( - result.changed_count or result.unknown_count or result.error_count - ): - return 1 - return 1 if request.fail_on_error and (result.unknown_count or result.error_count) else 0 - - -def _format_freshness_for_terminal(*, result: FreshnessCommandResult, no_color: bool) -> str: - return format_freshness_text( - result=result, - use_color=not no_color and supports_color(), - ) - - -def _source_freshness_observed_at() -> datetime: - return datetime.now(UTC) diff --git a/src/sqlbuild/cli/commands/main/execution/_test.py b/src/sqlbuild/cli/commands/main/execution/_test.py index c28661b7aa..52114df624 100644 --- a/src/sqlbuild/cli/commands/main/execution/_test.py +++ b/src/sqlbuild/cli/commands/main/execution/_test.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import write_execution_header from sqlbuild.cli.commands._helpers.test.execution import ( execute_test_plan, prepare_test_execution, @@ -19,7 +20,6 @@ TestExecutionPreparation, TestInvocation, ) -from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.cli.target_artifacts.main._write_test_runtime_target import write_test_runtime_target from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.testing.models import SqlTestExecutionResult diff --git a/src/sqlbuild/cli/commands/main/execution/command_warehouse.py b/src/sqlbuild/cli/commands/main/execution/command_warehouse.py deleted file mode 100644 index 27fdc6755f..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/command_warehouse.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Command-group warehouse public entry for cross-domain orchestration.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.commands._helpers.runtime.warehouses import ( - apply_command_warehouse as _apply_command_warehouse, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - - -def apply_command_warehouse( - *, - config: dict[str, object], - adapter_name: str, - project_dir: Path | None, - discovered_inputs: DiscoveredProjectInputs | None, - selected_target: str | None, -) -> dict[str, object]: - """Return connection config whose warehouse follows the active command's group.""" - - return _apply_command_warehouse( - config=config, - adapter_name=adapter_name, - project_dir=project_dir, - discovered_inputs=discovered_inputs, - selected_target=selected_target, - ) diff --git a/src/sqlbuild/cli/commands/main/execution/connection_progress.py b/src/sqlbuild/cli/commands/main/execution/connection_progress.py index 9efdfcb09b..3d6ae93e2a 100644 --- a/src/sqlbuild/cli/commands/main/execution/connection_progress.py +++ b/src/sqlbuild/cli/commands/main/execution/connection_progress.py @@ -4,7 +4,7 @@ from typing import TextIO -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter +from sqlbuild._native import ConnectionProgressReporter def build_connection_progress_reporter( diff --git a/src/sqlbuild/cli/commands/main/inspection/_debug.py b/src/sqlbuild/cli/commands/main/inspection/_debug.py deleted file mode 100644 index 0aef3e08b4..0000000000 --- a/src/sqlbuild/cli/commands/main/inspection/_debug.py +++ /dev/null @@ -1,37 +0,0 @@ -"""CLI debug command entry point.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from sqlbuild.cli.commands._helpers.debug.checks import build_debug_result -from sqlbuild.cli.commands._helpers.debug.output import format_debug_json, format_debug_text -from sqlbuild.cli.commands.models import DebugResult -from sqlbuild.presentation.main.supports_color import supports_color - - -def run_debug( - *, - project_dir: Path | None, - no_color: bool = False, - no_connection: bool = False, - selected_target: str | None = None, - json_output: bool = False, -) -> int: - """Validate project config and the active connection.""" - - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - result: DebugResult = build_debug_result( - project_dir=effective_project_dir, - check_connection=not no_connection, - selected_target=selected_target, - ) - if json_output: - sys.stdout.write(format_debug_json(result) + "\n") - else: - sys.stdout.write( - format_debug_text(result=result, use_color=(not no_color) and supports_color()) - ) - sys.stdout.flush() - return 0 if result.success else 1 diff --git a/src/sqlbuild/cli/commands/main/inspection/_lineage.py b/src/sqlbuild/cli/commands/main/inspection/_lineage.py deleted file mode 100644 index 94d02ab13d..0000000000 --- a/src/sqlbuild/cli/commands/main/inspection/_lineage.py +++ /dev/null @@ -1,12 +0,0 @@ -"""CLI lineage command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.lineage.execution import execute_lineage -from sqlbuild.cli.commands.models import LineageCommandRequest - - -def run_lineage(request: LineageCommandRequest) -> int: - """Execute the lineage command.""" - - return execute_lineage(request=request) diff --git a/src/sqlbuild/cli/commands/main/inspection/_query.py b/src/sqlbuild/cli/commands/main/inspection/_query.py index 4c5ad455a2..ff44bbbeb3 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_query.py +++ b/src/sqlbuild/cli/commands/main/inspection/_query.py @@ -5,15 +5,15 @@ import sys from pathlib import Path -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import ( + discover_project_inputs, + resolve_project_connection_config, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import QueryResult from sqlbuild.cli.commands._helpers.query.output import render_query_result from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, -) from sqlbuild.cli.commands.exceptions import CliUserError, QueryExecutionError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( diff --git a/src/sqlbuild/cli/commands/main/inspection/_scope.py b/src/sqlbuild/cli/commands/main/inspection/_scope.py deleted file mode 100644 index 5960b6efff..0000000000 --- a/src/sqlbuild/cli/commands/main/inspection/_scope.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Offline declaration-scope inspection command.""" - -from sqlbuild.cli.commands._helpers.scope.command import run_scope_command -from sqlbuild.cli.commands.models import ScopeCommandRequest - - -def run_scope(*, request: ScopeCommandRequest) -> int: - """Run one offline scope inspection.""" - - return run_scope_command(request=request) diff --git a/src/sqlbuild/cli/commands/main/project/_refactor.py b/src/sqlbuild/cli/commands/main/project/_refactor.py deleted file mode 100644 index 9752d45334..0000000000 --- a/src/sqlbuild/cli/commands/main/project/_refactor.py +++ /dev/null @@ -1,10 +0,0 @@ -"""CLI entry for `sqb rename` and `sqb mv`.""" - -from sqlbuild.cli.commands._helpers.refactor.command import run_refactor_command -from sqlbuild.cli.commands.models import RefactorCommandRequest - - -def run_refactor(*, request: RefactorCommandRequest) -> int: - """Run one rename or move.""" - - return run_refactor_command(request=request) diff --git a/src/sqlbuild/cli/commands/main/state/_janitor.py b/src/sqlbuild/cli/commands/main/state/_janitor.py deleted file mode 100644 index 38050952da..0000000000 --- a/src/sqlbuild/cli/commands/main/state/_janitor.py +++ /dev/null @@ -1,83 +0,0 @@ -"""CLI janitor command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.janitor_output.outputs import ( - confirm_janitor_plan, - janitor_plan_has_work, - write_janitor_cancelled, - write_janitor_completion, - write_janitor_disabled, - write_janitor_plan, - write_janitor_preview_complete, - write_janitor_preview_start, -) -from sqlbuild.cli.commands._helpers.janitor_runtime.compilation import compile_janitor_project -from sqlbuild.cli.commands._helpers.janitor_runtime.connections import ( - close_janitor_warehouse, - connect_janitor_warehouse, -) -from sqlbuild.cli.commands._helpers.janitor_runtime.execution import execute_janitor_cleanup -from sqlbuild.cli.commands._helpers.janitor_runtime.invocation import ( - resolve_janitor_invocation, - resolve_janitor_settings, -) -from sqlbuild.cli.commands._helpers.janitor_runtime.planning import ( - build_janitor_execution_plan, -) -from sqlbuild.cli.commands.models import ( - JanitorCommandRequest, - JanitorCompileContext, - JanitorConnectionContext, - JanitorInvocation, - JanitorPlanningResult, - JanitorSettings, -) -from sqlbuild.executor.janitor.models import JanitorExecutionResult - - -def run_janitor(request: JanitorCommandRequest) -> int: - """Execute the janitor command.""" - - invocation: JanitorInvocation = resolve_janitor_invocation(request=request) - if not invocation.discovered_inputs.project_config.janitor.enabled: - write_janitor_disabled(invocation=invocation) - return 0 - settings: JanitorSettings = resolve_janitor_settings(request=request, invocation=invocation) - if invocation.as_target is not None: - write_janitor_preview_start(invocation=invocation) - compile_context: JanitorCompileContext = compile_janitor_project(invocation=invocation) - connection_context: JanitorConnectionContext = connect_janitor_warehouse( - invocation=invocation, - compile_context=compile_context, - ) - try: - planning_result: JanitorPlanningResult = build_janitor_execution_plan( - invocation=invocation, - settings=settings, - compile_context=compile_context, - connection_context=connection_context, - ) - write_janitor_plan(invocation=invocation, planning_result=planning_result) - if invocation.as_target is not None: - write_janitor_preview_complete(invocation=invocation) - return 1 if planning_result.plan.blocked_schemas else 0 - if planning_result.plan.blocked_schemas: - return 1 - if not janitor_plan_has_work(planning_result): - return 0 - if not request.auto_approve and not confirm_janitor_plan(planning_result=planning_result): - write_janitor_cancelled() - return 1 - result: JanitorExecutionResult = execute_janitor_cleanup( - compile_context=compile_context, - connection_context=connection_context, - planning_result=planning_result, - ) - write_janitor_completion(invocation=invocation, result=result) - return 0 - finally: - _ = close_janitor_warehouse( - compile_context=compile_context, - connection_context=connection_context, - ) diff --git a/src/sqlbuild/cli/commands/main/workspace/_init.py b/src/sqlbuild/cli/commands/main/workspace/_init.py index ae43d974da..3ad4cb9877 100644 --- a/src/sqlbuild/cli/commands/main/workspace/_init.py +++ b/src/sqlbuild/cli/commands/main/workspace/_init.py @@ -4,9 +4,8 @@ from pathlib import Path +from sqlbuild._native import CliDocument, update_sqlbuild_skills from sqlbuild.cli.commands._helpers.init.scaffold import scaffold_blank_project -from sqlbuild.cli.commands._helpers.skills.update import update_sqlbuild_skills -from sqlbuild.presentation.classes.cli_document import CliDocument from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/main/workspace/_playground.py b/src/sqlbuild/cli/commands/main/workspace/_playground.py deleted file mode 100644 index 809697bab5..0000000000 --- a/src/sqlbuild/cli/commands/main/workspace/_playground.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Create a local SQLBuild playground project.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.playground.completion_output import ( - render_playground_completion_text, -) -from sqlbuild.cli.commands._helpers.playground.scaffold import ( - resolve_playground_target, - write_playground_project, -) -from sqlbuild.cli.commands.models import ( - PlaygroundCommandRequest, - PlaygroundTarget, -) - - -def run_playground(request: PlaygroundCommandRequest) -> int: - """Create a self-contained waffle shop playground project.""" - - target: PlaygroundTarget = resolve_playground_target(request=request) - write_playground_project(target=target) - completion_text: str = render_playground_completion_text(request=request, target=target) - print(completion_text, end="") - return 0 diff --git a/src/sqlbuild/cli/commands/main/workspace/_skills.py b/src/sqlbuild/cli/commands/main/workspace/_skills.py deleted file mode 100644 index f380857453..0000000000 --- a/src/sqlbuild/cli/commands/main/workspace/_skills.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Install or update SQLBuild agent skills.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.commands._helpers.skills.update import update_sqlbuild_skills -from sqlbuild.cli.output.models import ( - SkillUpdateResult, -) - - -def run_skills_update( - *, - project_dir: Path | None, - global_install: bool = False, - targets: tuple[str, ...] = (), - force: bool = False, -) -> int: - """Install or update SQLBuild skill files.""" - - base_dir: Path = project_dir if project_dir is not None else Path.cwd() - result: SkillUpdateResult = update_sqlbuild_skills( - project_dir=base_dir, - global_install=global_install, - requested_targets=targets, - force=force, - ) - print("Updated SQLBuild skill files:") - for written_path in result.written_paths: - print(f" {written_path}") - return 0 diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 4604d9625a..654d4ccd72 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -9,8 +9,6 @@ from typing import TYPE_CHECKING, Any, TextIO from sqlbuild.cli.commands.types import ( - DebugCheckStatus, - FreshnessSourceStatus, PlaygroundTemplate, ) from sqlbuild.cli.compile.models import ( # noqa: F401 @@ -27,15 +25,12 @@ SelectorFileSummary, ) from sqlbuild.cli.output.models import ( # noqa: F401 - SkillInstallTarget, SkillMaintenanceResult, - SkillSettings, SkillUpdateResult, WrittenTarget, ) from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts # noqa: F401 from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus from sqlbuild.cost.types import CostStatus from sqlbuild.spec.contracts.models import ( CostConfig, @@ -45,27 +40,25 @@ ) if TYPE_CHECKING: + from sqlbuild._native import ( + ConnectionProgressReporter, + PlanningProgressReporter, + ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands.classes.build_progress_callbacks import BuildProgressCallbacks from sqlbuild.cli.commands.classes.direct_python_lifecycle_state import ( DirectPythonLifecycleState, ) from sqlbuild.cli.progress.classes.audit_progress_reporter import AuditProgressReporter - from sqlbuild.cli.progress.classes.connection_progress_reporter import ( - ConnectionProgressReporter, - ) from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( NestedCommandProgressCallbacks, ) - from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, CompiledSqlScenario, CompilerDiagnostic, ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs - from sqlbuild.compiler.lineage.models import ColumnLineageEdge, QualifiedLineageColumn from sqlbuild.compiler.pipeline.models import ( ClonePipelineResult, CompilePipelineResult, @@ -432,30 +425,6 @@ class DbtInitInvocation: use_color: bool -@dataclass(frozen=True) -class DebugLine: - label: str - message: str - status: DebugCheckStatus | None = None - status_message: str | None = None - - -@dataclass(frozen=True) -class DebugResult: - runtime: tuple[DebugLine, ...] - configuration: tuple[DebugLine, ...] - providers: tuple[DebugLine, ...] - connection: tuple[DebugLine, ...] - - @property - def success(self) -> bool: - return all(line.status != DebugCheckStatus.ERROR for line in self.lines) - - @property - def lines(self) -> tuple[DebugLine, ...]: - return self.runtime + self.configuration + self.providers + self.connection - - @dataclass(frozen=True) class DiffCommandRequest: """CLI inputs for one diff command invocation.""" @@ -587,79 +556,6 @@ class FreshnessCommandRequest: fail_on_stale: bool = False -@dataclass(frozen=True) -class FreshnessSourceResult: - """Freshness observation result for one source.""" - - name: str - status: FreshnessSourceStatus - strategy: str | None = None - value_kind: str | None = None - current_data_version: str | None = None - previous_data_version: str | None = None - lag_tolerance: str | None = None - target_database: str | None = None - target_schema: str | None = None - target_name: str | None = None - message: str | None = None - age_status: SourceFreshnessAgeStatus | None = None - - -@dataclass(frozen=True) -class FreshnessCommandResult: - """Source freshness command output payload.""" - - sources: tuple[FreshnessSourceResult, ...] = field(default_factory=tuple) - - @property - def observed_count(self) -> int: - return sum(1 for source in self.sources if source.status == FreshnessSourceStatus.OBSERVED) - - @property - def changed_count(self) -> int: - return sum(1 for source in self.sources if source.status == FreshnessSourceStatus.CHANGED) - - @property - def unchanged_count(self) -> int: - return sum(1 for source in self.sources if source.status == FreshnessSourceStatus.UNCHANGED) - - @property - def tolerated_count(self) -> int: - return sum(1 for source in self.sources if source.status == FreshnessSourceStatus.TOLERATED) - - @property - def unknown_count(self) -> int: - return sum(1 for source in self.sources if source.status == FreshnessSourceStatus.UNKNOWN) - - @property - def error_count(self) -> int: - return sum(1 for source in self.sources if source.status == FreshnessSourceStatus.ERROR) - - @property - def age_pass_count(self) -> int: - return sum( - 1 for source in self.sources if source.age_status == SourceFreshnessAgeStatus.PASS - ) - - @property - def age_warn_count(self) -> int: - return sum( - 1 for source in self.sources if source.age_status == SourceFreshnessAgeStatus.WARN - ) - - @property - def age_error_count(self) -> int: - return sum( - 1 for source in self.sources if source.age_status == SourceFreshnessAgeStatus.ERROR - ) - - @property - def age_unknown_count(self) -> int: - return sum( - 1 for source in self.sources if source.age_status == SourceFreshnessAgeStatus.UNKNOWN - ) - - @dataclass(frozen=True) class JanitorCommandRequest: """CLI inputs for one janitor command invocation.""" @@ -720,70 +616,6 @@ class JanitorPlanningResult: plan: JanitorPlan -@dataclass(frozen=True) -class LineageNode: - """One displayable lineage graph node.""" - - key: CompiledObjectKey - relative_path: str | None = None - qualified_name: str | None = None - - -@dataclass(frozen=True) -class LineageGraph: - """Selected lineage graph slice.""" - - nodes: tuple[LineageNode, ...] - edges: tuple[tuple[CompiledObjectKey, CompiledObjectKey], ...] - focus_keys: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) - direction: str | None = None - - -@dataclass(frozen=True) -class RelationLineageIndex: - """Cacheable structural graph used by relation-level lineage inspection.""" - - nodes: dict[CompiledObjectKey, LineageNode] - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] - tag_index: dict[str, frozenset[CompiledObjectKey]] - path_index: dict[CompiledObjectKey, str] - all_keys: dict[str, CompiledObjectKey] - - -@dataclass(frozen=True) -class LineageSelectionAnchors: - """Selector anchors used for optional post-selection depth trimming.""" - - upstream: frozenset[CompiledObjectKey] = field(default_factory=frozenset) - downstream: frozenset[CompiledObjectKey] = field(default_factory=frozenset) - retained: frozenset[CompiledObjectKey] = field(default_factory=frozenset) - - -@dataclass(frozen=True) -class LineageTarget: - """One positional target, `[kind:]` or `[kind:].`, resolved by name.""" - - kind: str | None - name: str - key: CompiledObjectKey | None - column_name: str | None - kind_matches: bool - - -@dataclass(frozen=True) -class ColumnLineageTrace: - """Selected column-level lineage trace.""" - - target: QualifiedLineageColumn - trace: tuple[ColumnLineageEdge, ...] - direction: str - mode: ColumnLineageMode = ColumnLineageMode.RICH - max_depth: int | None = None - analyzed_model_count: int = 0 - truncated: bool = False - - @dataclass(frozen=True) class LoadCommandRequest: """CLI inputs for one load command invocation.""" @@ -908,14 +740,6 @@ class PlaygroundCommandRequest: template: str = PlaygroundTemplate.WAFFLE_SHOP.value -@dataclass(frozen=True) -class PlaygroundTarget: - """Resolved playground destination directory and template.""" - - target_dir: Path - template: PlaygroundTemplate - - @dataclass(frozen=True) class AdapterConnectionContext: """Resolved adapter and connection configuration for one CLI command.""" @@ -1095,16 +919,6 @@ class RefactorCommandRequest: no_color: bool = False -@dataclass(frozen=True) -class LayerMoveSuggestion: - """The one-step `sqb mv` for a rename refused only by the layer-folder rules.""" - - new_name: str - folder: str | None - layer_folder: str | None - command: str - - @dataclass(frozen=True) class RefactorCompile: """One offline compile for a refactoring: its facts, or the errors that stopped it.""" diff --git a/src/sqlbuild/cli/commands/types.py b/src/sqlbuild/cli/commands/types.py index ac84926844..1927cab32a 100644 --- a/src/sqlbuild/cli/commands/types.py +++ b/src/sqlbuild/cli/commands/types.py @@ -19,12 +19,6 @@ class CompileLineageMode(StrEnum): NONE = "none" -class DebugCheckStatus(StrEnum): - OK = "OK" - ERROR = "ERROR" - SKIP = "SKIP" - - class CliCommand(StrEnum): COMPILE = "compile" DAG = "dag" @@ -132,17 +126,6 @@ def __call__( ) -> int: ... -class FreshnessSourceStatus(StrEnum): - """Command status for one source freshness observation.""" - - OBSERVED = "observed" - CHANGED = "changed" - UNCHANGED = "unchanged" - TOLERATED = "tolerated" - UNKNOWN = "unknown" - ERROR = "error" - - class PlaygroundTemplate(StrEnum): """Available `sqb playground` templates.""" diff --git a/src/sqlbuild/cli/output/_helpers/integration_identity.py b/src/sqlbuild/cli/output/_helpers/integration_identity.py deleted file mode 100644 index 1c245af6ef..0000000000 --- a/src/sqlbuild/cli/output/_helpers/integration_identity.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Canonical integration resource identity resolution.""" - -from __future__ import annotations - -from sqlbuild.cli.output.constants import INTEGRATION_CHECK_KINDS, INTEGRATION_LOADER_KIND - -_MODEL_RESOURCE_KINDS: frozenset[str] = frozenset({"custom", "model", "snapshot", "table", "view"}) - - -def integration_resource_id( - *, resource_kind: str, resource_name: str, check_id: str | None, loader_name: str | None = None -) -> str: - """Return the canonical resource identifier for an integration result.""" - - if resource_kind in INTEGRATION_CHECK_KINDS: - return check_id or "" - if resource_kind in _MODEL_RESOURCE_KINDS: - return f"model:{resource_name}" - if resource_kind == INTEGRATION_LOADER_KIND and loader_name is not None: - return f"source:{resource_name}" - return f"{resource_kind}:{resource_name}" diff --git a/src/sqlbuild/cli/output/_helpers/integration_result.py b/src/sqlbuild/cli/output/_helpers/integration_result.py deleted file mode 100644 index 9cbd71605d..0000000000 --- a/src/sqlbuild/cli/output/_helpers/integration_result.py +++ /dev/null @@ -1,512 +0,0 @@ -"""Canonical integration-result projection from executor result models.""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from dataclasses import replace -from typing import Any, cast - -from sqlbuild._native import serialize_future_cursor_safety, serialize_maximum_start_safety -from sqlbuild.cli.output._helpers.integration_identity import integration_resource_id -from sqlbuild.cli.output._helpers.integration_validation import encode_integration_json -from sqlbuild.cli.output.constants import ( - INTEGRATION_RESOURCE_FAILED_EVENT, - INTEGRATION_RESULT_RECORD_KIND, - INTEGRATION_RESULT_SCHEMA_VERSION, -) -from sqlbuild.cli.output.models import ( - IntegrationAssetResult, - IntegrationCheckResult, - IntegrationResultEnvelope, -) -from sqlbuild.cli.output.types import IntegrationOutputKind -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditSeverity -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build.models import ( - FunctionExecutionResult, - SeedExecutionResult, -) -from sqlbuild.executor.clone.models import CloneItemResult -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.python_nodes.models import ( - PythonCheckExecutionResult, - PythonNodeExecutionResult, -) -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.testing.main.resource_id import sql_test_resource_id -from sqlbuild.executor.testing.models import SqlTestExecutionResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.runtime.observability.constants import RESOURCE_ATTEMPT_SKIPPED_EVENT -from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError -from sqlbuild.runtime.observability.models import LifecycleEvent -from sqlbuild.runtime.observability.types import JSONValue - - -def build_integration_result( - *, - result: object, - terminal: LifecycleEvent, - event_sequence: int, - plan: PlanOutput | None, - command: str, -) -> IntegrationResultEnvelope | None: - """Build one canonical terminal/result envelope for an integration callback.""" - - asset: IntegrationAssetResult | None = _asset_result(result=result, plan=plan) - if asset is not None: - payload_kind: object = terminal.payload.get("resource_kind") - payload_name: object = terminal.payload.get("resource_name") - asset = replace( - asset, - kind=payload_kind if isinstance(payload_kind, str) else "", - name=payload_name if isinstance(payload_name, str) else "", - status=_canonical_status(terminal=terminal, fallback=asset.status), - ) - checks: tuple[IntegrationCheckResult, ...] = _check_results(result=result) - if asset is None and not checks: - return None - output_kind: IntegrationOutputKind = ( - IntegrationOutputKind.ASSET if asset is not None else IntegrationOutputKind.CHECK - ) - payload: Mapping[str, JSONValue] = terminal.payload - return IntegrationResultEnvelope( - schema_version=INTEGRATION_RESULT_SCHEMA_VERSION, - record_kind=INTEGRATION_RESULT_RECORD_KIND, - event_id=terminal.event_id, - event_sequence=event_sequence, - event_type=terminal.event_type, - occurred_at=terminal.occurred_at.isoformat(), - invocation_id=terminal.invocation_id, - run_id=terminal.run_id or "", - resource_id=terminal.resource_id or "", - resource_attempt_id=terminal.resource_attempt_id or "", - operation_id=terminal.operation_id, - statement_id=terminal.statement_id, - resource_kind=str(payload.get("resource_kind") or ""), - resource_name=str(payload.get("resource_name") or ""), - attempt_number=_attempt_number(payload.get("attempt_number")), - duration_ms=_number(payload.get("duration_ms")), - output_kind=output_kind, - command=command, - error_code=_optional_text(payload.get("error_code")), - error_type=_optional_text(payload.get("error_type")), - skip_code=_optional_text(payload.get("skip_code")), - skip_mode=_optional_text(payload.get("skip_mode")), - asset=asset, - checks=checks, - ) - - -def build_clone_integration_result( - *, item: CloneItemResult, terminal: LifecycleEvent, event_sequence: int -) -> IntegrationResultEnvelope: - """Build one canonical clone terminal/result envelope.""" - - payload: Mapping[str, JSONValue] = terminal.payload - return IntegrationResultEnvelope( - schema_version=INTEGRATION_RESULT_SCHEMA_VERSION, - record_kind=INTEGRATION_RESULT_RECORD_KIND, - event_id=terminal.event_id, - event_sequence=event_sequence, - event_type=terminal.event_type, - occurred_at=terminal.occurred_at.isoformat(), - invocation_id=terminal.invocation_id, - run_id=terminal.run_id or "", - resource_id=terminal.resource_id or "", - resource_attempt_id=terminal.resource_attempt_id or "", - operation_id=terminal.operation_id, - statement_id=terminal.statement_id, - resource_kind=str(payload.get("resource_kind") or ""), - resource_name=str(payload.get("resource_name") or ""), - attempt_number=_attempt_number(payload.get("attempt_number")), - duration_ms=_number(payload.get("duration_ms")), - output_kind=IntegrationOutputKind.ASSET, - command="clone", - error_code=_optional_text(payload.get("error_code")), - error_type=_optional_text(payload.get("error_type")), - skip_code=_optional_text(payload.get("skip_code")), - skip_mode=_optional_text(payload.get("skip_mode")), - asset=IntegrationAssetResult( - kind=str(payload.get("resource_kind") or ""), - name=str(payload.get("resource_name") or ""), - status=_canonical_status(terminal=terminal, fallback=item.status.value), - action=item.action.value, - target=item.destination_relation, - origin_relation=item.origin_relation, - ), - ) - - -def build_degraded_integration_result( - *, - result: object, - terminal: LifecycleEvent, - event_sequence: int, - command: str, -) -> IntegrationResultEnvelope | None: - """Build a minimal result after optional integration enrichment fails.""" - - payload: Mapping[str, JSONValue] = terminal.payload - resource_kind: str = str(payload.get("resource_kind") or "") - resource_name: str = str(payload.get("resource_name") or "") - asset: IntegrationAssetResult | None = None - checks: tuple[IntegrationCheckResult, ...] = () - if isinstance( - result, - ( - ModelExecutionResult, - SeedExecutionResult, - FunctionExecutionResult, - LoadExecutionResult, - PythonNodeExecutionResult, - CloneItemResult, - ), - ): - asset = IntegrationAssetResult( - kind=resource_kind, - name=resource_name, - status=_terminal_status(terminal), - ) - elif isinstance( - result, - (AuditExecutionResult, SqlTestExecutionResult, PythonCheckExecutionResult), - ): - checks = ( - IntegrationCheckResult( - kind=resource_kind, - name=resource_name, - check_id=terminal.resource_id or "", - passed=terminal.event_type != INTEGRATION_RESOURCE_FAILED_EVENT, - status=( - "error" if terminal.event_type == INTEGRATION_RESOURCE_FAILED_EVENT else "pass" - ), - ), - ) - if asset is None and not checks: - return None - return IntegrationResultEnvelope( - schema_version=INTEGRATION_RESULT_SCHEMA_VERSION, - record_kind=INTEGRATION_RESULT_RECORD_KIND, - event_id=terminal.event_id, - event_sequence=event_sequence, - event_type=terminal.event_type, - occurred_at=terminal.occurred_at.isoformat(), - invocation_id=terminal.invocation_id, - run_id=terminal.run_id or "", - resource_id=terminal.resource_id or "", - resource_attempt_id=terminal.resource_attempt_id or "", - operation_id=terminal.operation_id, - statement_id=terminal.statement_id, - resource_kind=resource_kind, - resource_name=resource_name, - attempt_number=_attempt_number(payload.get("attempt_number")), - duration_ms=_number(payload.get("duration_ms")), - output_kind=( - IntegrationOutputKind.ASSET if asset is not None else IntegrationOutputKind.CHECK - ), - command=command, - error_code=_optional_text(payload.get("error_code")), - error_type=_optional_text(payload.get("error_type")), - skip_code=_optional_text(payload.get("skip_code")), - skip_mode=_optional_text(payload.get("skip_mode")), - projection_degraded=True, - asset=asset, - checks=checks, - ) - - -def result_resource_identity(*, result: object) -> tuple[str | None, str | None]: - """Return the canonical resource name and identifier for an executor result.""" - - if isinstance(result, ModelExecutionResult): - return result.model_name, integration_resource_id( - resource_kind="model", resource_name=result.model_name, check_id=None - ) - if isinstance(result, SeedExecutionResult): - return result.seed_name, integration_resource_id( - resource_kind="seed", resource_name=result.seed_name, check_id=None - ) - if isinstance(result, FunctionExecutionResult): - return result.function_name, integration_resource_id( - resource_kind=result.function_kind, - resource_name=result.function_name, - check_id=None, - ) - if isinstance(result, LoadExecutionResult): - return result.source_name, integration_resource_id( - resource_kind="source", resource_name=result.source_name, check_id=None - ) - if isinstance(result, PythonNodeExecutionResult): - return result.node_name, integration_resource_id( - resource_kind=result.kind.value, resource_name=result.node_name, check_id=None - ) - if isinstance(result, PythonCheckExecutionResult): - check_id: str = f"check:{result.node_name}" - return result.node_name, integration_resource_id( - resource_kind="python_check", resource_name=result.node_name, check_id=check_id - ) - if isinstance(result, SqlTestExecutionResult): - check_id = _sql_test_check_id(result) - return result.test_name, integration_resource_id( - resource_kind="sql_test", resource_name=result.test_name, check_id=check_id - ) - if isinstance(result, AuditExecutionResult): - check_id = _audit_check_id(result) - return result.audit_name, integration_resource_id( - resource_kind="audit", resource_name=result.audit_name, check_id=check_id - ) - return None, None - - -def _asset_result(*, result: object, plan: PlanOutput | None) -> IntegrationAssetResult | None: - if isinstance(result, ModelExecutionResult): - target: str | None = None - if plan is not None and result.model_name in plan.model_locations: - target = plan.model_locations[result.model_name].qualified_name - return IntegrationAssetResult( - kind=CompiledResourceType.MODEL.value, - name=result.model_name, - status=result.status.value, - target=target or result.promoted_relation, - staging_relation=result.staging_relation, - failed_phase=result.failed_phase.value if result.failed_phase else None, - future_cursor_safety=_bounded_metadata( - serialize_future_cursor_safety(result.future_cursor_safety) or {} - ), - maximum_start_safety=_bounded_metadata( - serialize_maximum_start_safety(result.maximum_start_safety) or {} - ), - microbatch=_model_microbatch(result), - ) - if isinstance(result, SeedExecutionResult): - target = None - if plan is not None and result.seed_name in plan.seed_locations: - target = plan.seed_locations[result.seed_name].qualified_name - return IntegrationAssetResult( - kind=CompiledResourceType.SEED.value, - name=result.seed_name, - status=result.status.value, - target=target, - ) - if isinstance(result, FunctionExecutionResult): - target = None - if plan is not None and result.function_name in plan.function_locations: - target = plan.function_locations[result.function_name].qualified_name - return IntegrationAssetResult( - kind=result.function_kind, - name=result.function_name, - status=result.status.value, - target=target, - ) - if isinstance(result, LoadExecutionResult): - return IntegrationAssetResult( - kind=result.resource_kind.value, - name=result.source_name, - status=result.status.value, - target=result.target, - staging_relation=result.staging_relation, - loader=result.loader_name, - rows_loaded=result.rows_loaded, - ) - if isinstance(result, PythonNodeExecutionResult): - return IntegrationAssetResult( - kind=result.kind.value, - name=result.node_name, - status=result.status.value, - materialized=result.materialized, - ) - return None - - -def _check_results(*, result: object) -> tuple[IntegrationCheckResult, ...]: - if isinstance(result, AuditExecutionResult): - return (_audit_result(result),) - if isinstance(result, SqlTestExecutionResult): - asset_name: str | None = result.step_results[0].model_name if result.step_results else None - return ( - IntegrationCheckResult( - kind="sql_test", - name=result.test_name, - check_id=_sql_test_check_id(result), - dag_check_id=_sql_test_check_id(result), - passed=result.outcome == SqlTestOutcome.PASS, - status=result.outcome.value, - asset_name=asset_name, - ), - ) - if isinstance(result, PythonCheckExecutionResult): - status: str = "pass" if result.passed else "warn" if result.warned else "fail" - return ( - IntegrationCheckResult( - kind="python_check", - name=result.node_name, - check_id=f"check:{result.node_name}", - dag_check_id=f"check:{result.node_name}", - passed=result.passed, - status=status, - severity=AuditSeverity(result.severity.value), - ), - ) - return () - - -def _audit_result(result: AuditExecutionResult) -> IntegrationCheckResult: - return IntegrationCheckResult( - kind="audit", - name=result.audit_name, - check_id=_audit_check_id(result), - dag_check_id=_audit_check_id(result), - passed=result.outcome in {AuditOutcome.PASS, AuditOutcome.INSUFFICIENT}, - status=result.outcome.value, - severity=result.severity, - row_count=result.row_count, - attachment_kind=result.attachment_kind.value, - asset_name=result.attached_target_name, - attached_column_name=result.attached_column_name, - attached_target_name=result.attached_target_name, - run_scope_phase=result.run_scope_phase.value, - reused=result.reused, - evaluation_mode=result.evaluation_mode.value, - measured_value=result.measured_value, - sample_count=result.sample_count, - sample_unit=result.sample_unit, - minimum_samples=result.minimum_samples, - thresholds=_render_measurement_thresholds(result.thresholds), - evidence_count=( - len(result.evidence_rows) - if result.evaluation_mode == AuditEvaluationMode.MEASUREMENT - else None - ), - evidence_truncated=( - result.evidence_truncated - if result.evaluation_mode == AuditEvaluationMode.MEASUREMENT - else None - ), - audit_definition_name=result.audit_definition_name, - audit_description=result.audit_description, - ) - - -def _render_measurement_thresholds( - thresholds: MeasurementThresholds | None, -) -> Mapping[str, object] | None: - if thresholds is None: - return None - rendered: dict[str, object] = {} - for severity, bound in (("warn", thresholds.warn), ("error", thresholds.error)): - if bound is not None: - rendered[severity] = _render_measurement_threshold_bound(bound) - return rendered - - -def _render_measurement_threshold_bound(bound: MeasurementThresholdBound) -> Mapping[str, object]: - rendered: dict[str, object] = {"operator": bound.operator.value} - if bound.limit is not None: - rendered["limit"] = bound.limit - else: - rendered["lower"] = bound.lower - rendered["upper"] = bound.upper - return rendered - - -def _audit_check_id(result: AuditExecutionResult) -> str: - return audit_resource_id( - audit_name=result.audit_name, - attachment_kind=result.attachment_kind, - attached_target_kind=result.attached_target_kind, - attached_target_name=result.attached_target_name, - attached_column_name=result.attached_column_name, - ) - - -def _sql_test_check_id(result: SqlTestExecutionResult) -> str: - return sql_test_resource_id( - test_name=result.test_name, - source_path=result.source_path, - block_index=result.block_index, - case_name=result.case_name, - ) - - -def _bounded_metadata(value: Mapping[str, object]) -> Mapping[str, JSONValue]: - try: - encoded: str = encode_integration_json(value=value) - except ObservabilityValidationError: - return {} - decoded: Any = json.loads(encoded) - return cast(dict[str, JSONValue], decoded) - - -def _model_microbatch(result: ModelExecutionResult) -> Mapping[str, JSONValue]: - if result.microbatch_run_type is None: - return {} - values: dict[str, object] = { - "run_type": result.microbatch_run_type, - **( - {"strategy": result.microbatch_strategy} - if result.microbatch_strategy is not None - else {} - ), - **( - {"reason": result.microbatch_plan_reason} - if result.microbatch_plan_reason is not None - else {} - ), - "batch_count": result.batch_count, - "batch_size": result.batch_size, - "recovery_batch_count": result.microbatch_recovery_batch_count, - "known_gap_count": result.microbatch_known_gap_count, - "unaccounted_interval_count": result.microbatch_unaccounted_interval_count, - "synthetic_completion_count": result.microbatch_synthetic_completion_count, - "unknown_fingerprint_count": result.microbatch_unknown_fingerprint_count, - "contiguous_frontier": result.microbatch_contiguous_frontier, - "unaccounted_partition_policy": result.microbatch_unaccounted_partition_policy, - "replay_requirement_id": result.microbatch_replay_requirement_id, - "required_model_version_hash": result.microbatch_required_model_version_hash, - "physical_generation_id": result.microbatch_physical_generation_id, - "concurrent_enabled": result.microbatch_concurrent_enabled, - "batch_concurrency": result.microbatch_batch_concurrency, - "global_concurrency": result.microbatch_global_concurrency, - "replay_requirement_state": result.microbatch_replay_requirement_state, - "limit": result.microbatch_limit, - "count": result.microbatch_limit_count, - "action": ( - result.microbatch_limit_action.value - if result.microbatch_limit_action is not None - else None - ), - } - return _bounded_metadata({key: value for key, value in values.items() if value is not None}) - - -def _optional_text(value: object) -> str | None: - return value if isinstance(value, str) else None - - -def _canonical_status(*, terminal: LifecycleEvent, fallback: str) -> str: - if terminal.event_type == INTEGRATION_RESOURCE_FAILED_EVENT: - return "failed" - if terminal.event_type == RESOURCE_ATTEMPT_SKIPPED_EVENT: - return "skipped" - return fallback - - -def _terminal_status(terminal: LifecycleEvent) -> str: - if terminal.event_type == INTEGRATION_RESOURCE_FAILED_EVENT: - return "failed" - if terminal.event_type == RESOURCE_ATTEMPT_SKIPPED_EVENT: - return "skipped" - return "success" - - -def _attempt_number(value: object) -> int: - return value if isinstance(value, int) and not isinstance(value, bool) else 0 - - -def _number(value: object) -> int | float | None: - return value if isinstance(value, int | float) else None diff --git a/src/sqlbuild/cli/output/_helpers/integration_validation.py b/src/sqlbuild/cli/output/_helpers/integration_validation.py deleted file mode 100644 index 1e0510ae70..0000000000 --- a/src/sqlbuild/cli/output/_helpers/integration_validation.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Strict bounded validation and encoding for integration-result values.""" - -from __future__ import annotations - -import json -import math -from collections.abc import Mapping, Sequence - -from sqlbuild.cli.output.constants import ( - INTEGRATION_MAXIMUM_START_ACTIONS, - INTEGRATION_MAXIMUM_START_INPUT_KEYS, - INTEGRATION_MAXIMUM_START_KEYS, - INTEGRATION_MAXIMUM_START_REQUIRED_KEYS, - MAX_INTEGRATION_COLLECTION_ITEMS, - MAX_INTEGRATION_IDENTIFIER_CHARS, - MAX_INTEGRATION_METADATA_BYTES, - MAX_INTEGRATION_NESTING_DEPTH, - MAX_INTEGRATION_RECORD_BYTES, - MAX_INTEGRATION_STRING_CHARS, -) -from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError -from sqlbuild.runtime.observability.types import JSONValue - - -def encode_integration_json(*, value: object, record: bool = False) -> str: - """Encode a validated JSON value using deterministic strict JSON settings.""" - - validate_json_value(value=value, depth=0) - try: - encoded: str = json.dumps( - value, - separators=(",", ":"), - ensure_ascii=True, - allow_nan=False, - ) - except (TypeError, ValueError) as error: - raise ObservabilityValidationError("integration value is not strict JSON") from error - limit: int = MAX_INTEGRATION_RECORD_BYTES if record else MAX_INTEGRATION_METADATA_BYTES - if len(encoded.encode("utf-8")) > limit: - raise ObservabilityValidationError("integration JSON exceeds size limit") - return encoded - - -def validate_json_value(*, value: object, depth: int) -> None: - """Reject unsafe, non-JSON, deeply nested, or oversized integration values.""" - - if depth > MAX_INTEGRATION_NESTING_DEPTH: - raise ObservabilityValidationError("integration JSON exceeds nesting limit") - if value is None or type(value) is bool or type(value) is int: - return - if type(value) is float: - if not math.isfinite(value): - raise ObservabilityValidationError("integration JSON numbers must be finite") - return - if type(value) is str: - if len(value) > MAX_INTEGRATION_STRING_CHARS: - raise ObservabilityValidationError("integration JSON string exceeds size limit") - return - if isinstance(value, Mapping): - if len(value) > MAX_INTEGRATION_COLLECTION_ITEMS: - raise ObservabilityValidationError("integration JSON object exceeds item limit") - for key, item in value.items(): - if type(key) is not str: - raise ObservabilityValidationError("integration JSON object keys must be strings") - if not key or len(key) > MAX_INTEGRATION_IDENTIFIER_CHARS: - raise ObservabilityValidationError("integration JSON object key is invalid") - validate_json_value(value=item, depth=depth + 1) - return - if isinstance(value, Sequence) and not isinstance(value, bytes | bytearray): - if len(value) > MAX_INTEGRATION_COLLECTION_ITEMS: - raise ObservabilityValidationError("integration JSON array exceeds item limit") - for item in value: - validate_json_value(value=item, depth=depth + 1) - return - raise ObservabilityValidationError("integration value is not JSON") - - -def validate_identifier(*, value: object, field_name: str) -> None: - """Validate one required bounded integration identifier.""" - - if type(value) is not str or not value or len(value) > MAX_INTEGRATION_IDENTIFIER_CHARS: - raise ObservabilityValidationError(f"integration {field_name} is invalid") - - -def validate_optional_identifier(*, value: object, field_name: str) -> None: - """Validate one optional bounded integration identifier.""" - - if value is not None: - validate_identifier(value=value, field_name=field_name) - - -def validate_maximum_start_safety(value: Mapping[str, JSONValue]) -> None: - """Validate the exact bounded structure emitted by the shared final JSON serializer.""" - - validate_allowlisted_mapping( - value=value, - field_name="maximum_start_safety", - allowed_keys=INTEGRATION_MAXIMUM_START_KEYS, - ) - if not value: - return - if set(value) != INTEGRATION_MAXIMUM_START_REQUIRED_KEYS: - raise ObservabilityValidationError( - "integration maximum_start_safety must use the canonical structure" - ) - if value.get("action") not in INTEGRATION_MAXIMUM_START_ACTIONS: - raise ObservabilityValidationError("integration maximum_start_safety action is unsupported") - optional_text_fields: frozenset[str] = frozenset({"highest_eligible_target_max"}) - for field_name in INTEGRATION_MAXIMUM_START_REQUIRED_KEYS - {"action", "input"}: - field_value: object = value.get(field_name) - if type(field_value) is not str and not ( - field_name in optional_text_fields and field_value is None - ): - raise ObservabilityValidationError( - f"integration maximum_start_safety {field_name} has an invalid type" - ) - input_value: object = value.get("input") - if ( - not isinstance(input_value, Mapping) - or set(input_value) != INTEGRATION_MAXIMUM_START_INPUT_KEYS - ): - raise ObservabilityValidationError( - "integration maximum_start_safety input must use the canonical structure" - ) - if any(type(item) is not str or not item for item in input_value.values()): - raise ObservabilityValidationError( - "integration maximum_start_safety input values must be non-empty strings" - ) - - -def validate_allowlisted_mapping( - *, value: object, field_name: str, allowed_keys: frozenset[str] -) -> None: - """Validate one framework-generated bounded scalar mapping.""" - - if not isinstance(value, Mapping): - raise ObservabilityValidationError(f"integration {field_name} must be an object") - if any(type(key) is not str for key in value): - raise ObservabilityValidationError(f"integration {field_name} keys must be strings") - if not set(value).issubset(allowed_keys): - raise ObservabilityValidationError(f"integration {field_name} contains unknown fields") - _ = encode_integration_json(value=value) - _validate_nested_mapping_keys( - value=value, - field_name=field_name, - allowed_keys=allowed_keys, - ) - - -def _validate_nested_mapping_keys( - *, value: object, field_name: str, allowed_keys: frozenset[str] -) -> None: - if isinstance(value, Mapping): - if not set(value).issubset(allowed_keys): - raise ObservabilityValidationError(f"integration {field_name} contains unknown fields") - for item in value.values(): - _validate_nested_mapping_keys( - value=item, - field_name=field_name, - allowed_keys=allowed_keys, - ) - return - if isinstance(value, list | tuple): - for item in value: - _validate_nested_mapping_keys( - value=item, - field_name=field_name, - allowed_keys=allowed_keys, - ) diff --git a/src/sqlbuild/cli/output/classes/execution_event_writer.py b/src/sqlbuild/cli/output/classes/execution_event_writer.py deleted file mode 100644 index 7927d476de..0000000000 --- a/src/sqlbuild/cli/output/classes/execution_event_writer.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Thread-safe JSON Lines execution event writer.""" - -from __future__ import annotations - -import os -import sys -import threading -from contextlib import AbstractContextManager, nullcontext -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import ( - TerminalEventClaim, - TerminalEventIndex, - current_terminal_event_index, -) -from sqlbuild.cli.output._helpers.integration_result import ( - build_clone_integration_result, - build_degraded_integration_result, - build_integration_result, - result_resource_identity, -) -from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV -from sqlbuild.cli.output.models import IntegrationResultEnvelope -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.clone.models import CloneItemResult -from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError - - -class ExecutionEventWriter: - """Append and flush structured execution events for a parent integration.""" - - def __init__(self, *, path: Path | None = None) -> None: - configured_path: str | None = os.environ.get(INTEGRATION_RESULT_PATH_ENV) - self._path: Path | None = path or (Path(configured_path) if configured_path else None) - self._stream: TextIO | None = None - self._lock: threading.Lock = threading.Lock() - self._terminal_index: TerminalEventIndex | None = current_terminal_event_index() - self._closed: bool = False - if self._path is not None: - self._path.parent.mkdir(parents=True, exist_ok=True) - self._stream = self._path.open(mode="a", encoding="utf-8") - - def write_build_result( - self, *, result: object, plan: PlanOutput | None, command: str = "build" - ) -> None: - """Write canonical terminal evidence plus typed build-result enrichment.""" - - with self._lock: - if self._closed or self._stream is None: - return - with self._output_scope(): - resource_name, resource_id = result_resource_identity(result=result) - claim: TerminalEventClaim | None = ( - self._terminal_index.claim_resource_terminal( - resource_name=resource_name, - resource_id=resource_id, - ) - if self._terminal_index is not None and resource_name is not None - else None - ) - if claim is None: - return - try: - envelope: IntegrationResultEnvelope | None = build_integration_result( - result=result, - terminal=claim.terminal, - event_sequence=claim.event_sequence, - plan=plan, - command=command, - ) - self._write_envelope(envelope) - except ObservabilityValidationError as error: - self._warn_degraded_output( - error=error, - execution_succeeded=claim.terminal.event_type.endswith("completed"), - ) - try: - self._write_envelope( - build_degraded_integration_result( - result=result, - terminal=claim.terminal, - event_sequence=claim.event_sequence, - command=command, - ) - ) - except ObservabilityValidationError: - return - - def write_clone_result(self, *, item: CloneItemResult, resource_type: str) -> None: - """Write canonical terminal evidence plus typed clone-result enrichment.""" - - with self._lock: - if self._closed or self._stream is None: - return - with self._output_scope(): - claim: TerminalEventClaim | None = ( - self._terminal_index.claim_resource_terminal( - resource_name=item.name, - resource_id=f"{resource_type}:{item.name}", - ) - if self._terminal_index is not None - else None - ) - if claim is None: - return - try: - self._write_envelope( - build_clone_integration_result( - item=item, - terminal=claim.terminal, - event_sequence=claim.event_sequence, - ) - ) - except ObservabilityValidationError as error: - self._warn_degraded_output( - error=error, - execution_succeeded=claim.terminal.event_type.endswith("completed"), - ) - try: - self._write_envelope( - build_degraded_integration_result( - result=item, - terminal=claim.terminal, - event_sequence=claim.event_sequence, - command="clone", - ) - ) - except ObservabilityValidationError: - return - - def _write_envelope(self, envelope: IntegrationResultEnvelope | None) -> None: - if self._stream is None or envelope is None: - return - self._stream.write(envelope.to_json()) - self._stream.flush() - - def _output_scope(self) -> AbstractContextManager[object]: - if self._terminal_index is None: - return nullcontext() - return self._terminal_index.output_serialization_scope() - - @staticmethod - def _warn_degraded_output( - *, error: ObservabilityValidationError, execution_succeeded: bool - ) -> None: - completion: str = ( - "warehouse execution completed successfully" - if execution_succeeded - else "execution result remains authoritative" - ) - sys.stderr.write(f"warning: integration output degraded: {error}; {completion}\n") - sys.stderr.flush() - - def close(self) -> None: - """Close the event stream when configured.""" - - with self._lock: - if self._closed: - return - with self._output_scope(): - self._closed = True - if self._stream is not None: - self._stream.close() - self._stream = None diff --git a/src/sqlbuild/cli/output/constants.py b/src/sqlbuild/cli/output/constants.py index bc2405aaa2..dd8a0acf62 100644 --- a/src/sqlbuild/cli/output/constants.py +++ b/src/sqlbuild/cli/output/constants.py @@ -1,169 +1,3 @@ """CLI execution output constants.""" -from sqlbuild.executor.clone.types import CloneAction -from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction - INTEGRATION_RESULT_PATH_ENV: str = "SQLBUILD_INTEGRATION_RESULT_PATH" -INTEGRATION_RESULT_SCHEMA_VERSION: int = 1 -INTEGRATION_RESULT_RECORD_KIND: str = "integration_result" -INTEGRATION_RESOURCE_FAILED_EVENT: str = "resource_attempt_failed" -INTEGRATION_CHECK_NON_FAIL_STATUSES: frozenset[str] = frozenset({"insufficient", "pass"}) -INTEGRATION_FAILED_STATUS: str = "failed" -INTEGRATION_SKIPPED_STATUS: str = "skipped" -INTEGRATION_LOADER_KIND: str = "loader" -INTEGRATION_AUDIT_KIND: str = "audit" -MAX_INTEGRATION_RECORD_BYTES: int = 65_536 -MAX_INTEGRATION_STRING_CHARS: int = 2_048 -MAX_INTEGRATION_IDENTIFIER_CHARS: int = 512 -MAX_INTEGRATION_COLLECTION_ITEMS: int = 128 -MAX_INTEGRATION_NESTING_DEPTH: int = 6 -MAX_INTEGRATION_METADATA_BYTES: int = 16_384 -INTEGRATION_ASSET_KINDS: frozenset[str] = frozenset( - { - "asset", - "custom", - "loader", - "model", - "seed", - "snapshot", - "source", - "table", - "table_fn", - "task", - "udf", - "view", - } -) -INTEGRATION_CHECK_KINDS: frozenset[str] = frozenset( - {INTEGRATION_AUDIT_KIND, "python_check", "sql_test"} -) -INTEGRATION_RESOURCE_KINDS: frozenset[str] = INTEGRATION_ASSET_KINDS | frozenset( - {"audit", "check", "test"} -) -INTEGRATION_ASSET_STATUSES: frozenset[str] = frozenset({"failed", "skipped", "success", "warning"}) -INTEGRATION_FAILED_PHASES: frozenset[str] = frozenset( - { - "audit", - "contract", - "custom_materialization", - "dml", - "fingerprint", - "microbatch_state", - "post_hook", - "pre_hook", - "promotion", - "schema_change", - "staging", - "type_enforcement", - } -) -INTEGRATION_CHECK_STATUSES: frozenset[str] = frozenset( - {"error", "fail", "insufficient", "pass", "warn"} -) -INTEGRATION_CHECK_SEVERITIES: frozenset[str] = frozenset({"error", "warn"}) -INTEGRATION_CHECK_ATTACHMENT_KINDS: frozenset[str] = frozenset({"end", "model", "source"}) -INTEGRATION_CHECK_RUN_SCOPE_PHASES: frozenset[str] = frozenset({"delta_and_final", "final"}) -INTEGRATION_CLONE_ACTIONS: frozenset[str] = frozenset(action.value for action in CloneAction) -INTEGRATION_SKIP_MODES: frozenset[str] = frozenset({"hard", "soft"}) -INTEGRATION_FUTURE_CURSOR_KEYS: frozenset[str] = frozenset( - { - "action", - "applied_bounds", - "cursor_column", - "determining_input", - "discovered_bounds", - "end", - "future_end_detected", - "future_start_detected", - "inputs", - "invocation_time", - "max_distance", - "maximum", - "maximum_allowed_bounds", - "minimum", - "relation", - "start", - } -) -INTEGRATION_MAXIMUM_START_KEYS: frozenset[str] = frozenset( - { - "action", - "cursor_column", - "effective_start", - "highest_eligible_target_max", - "input", - "invocation_time", - "max_ahead", - "maximum_allowed_start", - "physical_target_max", - "relation", - } -) -INTEGRATION_MAXIMUM_START_ACTIONS: frozenset[str] = frozenset( - action.value for action in FutureCursorAction -) -INTEGRATION_MAXIMUM_START_INPUT_KEYS: frozenset[str] = frozenset({"relation", "cursor_column"}) -INTEGRATION_MAXIMUM_START_REQUIRED_KEYS: frozenset[str] = frozenset( - { - "action", - "max_ahead", - "invocation_time", - "physical_target_max", - "highest_eligible_target_max", - "effective_start", - "maximum_allowed_start", - "input", - } -) -INTEGRATION_MICROBATCH_KEYS: frozenset[str] = frozenset( - { - "action", - "batch_concurrency", - "batch_count", - "batch_size", - "concurrent_enabled", - "contiguous_frontier", - "count", - "global_concurrency", - "known_gap_count", - "limit", - "physical_generation_id", - "recovery_batch_count", - "replay_requirement_id", - "replay_requirement_state", - "required_model_version_hash", - "run_type", - "strategy", - "reason", - "synthetic_completion_count", - "unaccounted_interval_count", - "unaccounted_partition_policy", - "unknown_fingerprint_count", - } -) -INTEGRATION_MICROBATCH_COUNT_KEYS: frozenset[str] = frozenset( - { - "batch_concurrency", - "batch_count", - "count", - "global_concurrency", - "known_gap_count", - "limit", - "recovery_batch_count", - "synthetic_completion_count", - "unaccounted_interval_count", - "unknown_fingerprint_count", - } -) -INTEGRATION_MICROBATCH_RUN_TYPES: frozenset[str] = frozenset( - {"backfill", "normal", "replay_on_change"} -) -INTEGRATION_MICROBATCH_LIMIT_ACTIONS: frozenset[str] = frozenset( - action.value for action in MicrobatchLimitAction -) -INTEGRATION_MICROBATCH_PARTITION_POLICIES: frozenset[str] = frozenset( - {"recover_all", "recover_empty", "synthesize"} -) -INTEGRATION_MICROBATCH_REPLAY_STATES: frozenset[str] = frozenset( - {"complete_with_unknown_fingerprints", "incomplete", "superseded", "verified_complete"} -) diff --git a/src/sqlbuild/cli/output/models.py b/src/sqlbuild/cli/output/models.py index 07ee050a4f..d40b3996ce 100644 --- a/src/sqlbuild/cli/output/models.py +++ b/src/sqlbuild/cli/output/models.py @@ -2,541 +2,10 @@ from __future__ import annotations -import json -import math -from collections.abc import Mapping -from dataclasses import asdict, dataclass, field -from datetime import datetime +from dataclasses import dataclass, field from pathlib import Path -from typing import Any, cast -from sqlbuild.cli.output._helpers.integration_identity import integration_resource_id -from sqlbuild.cli.output._helpers.integration_validation import ( - encode_integration_json, - validate_allowlisted_mapping, - validate_identifier, - validate_maximum_start_safety, - validate_optional_identifier, -) -from sqlbuild.cli.output.constants import ( - INTEGRATION_ASSET_KINDS, - INTEGRATION_ASSET_STATUSES, - INTEGRATION_AUDIT_KIND, - INTEGRATION_CHECK_ATTACHMENT_KINDS, - INTEGRATION_CHECK_KINDS, - INTEGRATION_CHECK_NON_FAIL_STATUSES, - INTEGRATION_CHECK_RUN_SCOPE_PHASES, - INTEGRATION_CHECK_SEVERITIES, - INTEGRATION_CHECK_STATUSES, - INTEGRATION_CLONE_ACTIONS, - INTEGRATION_FAILED_PHASES, - INTEGRATION_FAILED_STATUS, - INTEGRATION_FUTURE_CURSOR_KEYS, - INTEGRATION_MICROBATCH_COUNT_KEYS, - INTEGRATION_MICROBATCH_KEYS, - INTEGRATION_MICROBATCH_LIMIT_ACTIONS, - INTEGRATION_MICROBATCH_PARTITION_POLICIES, - INTEGRATION_MICROBATCH_REPLAY_STATES, - INTEGRATION_MICROBATCH_RUN_TYPES, - INTEGRATION_RESOURCE_FAILED_EVENT, - INTEGRATION_RESOURCE_KINDS, - INTEGRATION_RESULT_RECORD_KIND, - INTEGRATION_RESULT_SCHEMA_VERSION, - INTEGRATION_SKIP_MODES, - INTEGRATION_SKIPPED_STATUS, - MAX_INTEGRATION_COLLECTION_ITEMS, - MAX_INTEGRATION_RECORD_BYTES, -) -from sqlbuild.cli.output.types import ( - IntegrationOutputKind, - WrittenTargetDiagnostic, -) -from sqlbuild.compiler.auditing.types import ( - AuditEvaluationMode, - AuditSeverity, - ThresholdOperator, -) -from sqlbuild.runtime.observability.constants import ( - RESOURCE_ATTEMPT_SKIPPED_EVENT, - RESOURCE_SKIP_CODES, - RESOURCE_TERMINALS, -) -from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError -from sqlbuild.runtime.observability.types import JSONValue - - -@dataclass(frozen=True) -class IntegrationAssetResult: - """Bounded asset enrichment consumed by SQLBuild-owned integrations.""" - - kind: str - name: str - status: str - action: str | None = None - target: str | None = None - origin_relation: str | None = None - staging_relation: str | None = None - failed_phase: str | None = None - loader: str | None = None - rows_loaded: int | None = None - materialized: bool | None = None - future_cursor_safety: Mapping[str, JSONValue] = field(default_factory=dict) - maximum_start_safety: Mapping[str, JSONValue] = field(default_factory=dict) - microbatch: Mapping[str, JSONValue] = field(default_factory=dict) - - def __post_init__(self) -> None: - validate_identifier(value=self.kind, field_name="asset kind") - validate_identifier(value=self.name, field_name="asset name") - if self.kind not in INTEGRATION_ASSET_KINDS: - raise ObservabilityValidationError("integration asset kind is unsupported") - if self.status not in INTEGRATION_ASSET_STATUSES: - raise ObservabilityValidationError("integration asset status is unsupported") - if self.action is not None and self.action not in INTEGRATION_CLONE_ACTIONS: - raise ObservabilityValidationError("integration asset action is unsupported") - if self.failed_phase is not None and self.failed_phase not in INTEGRATION_FAILED_PHASES: - raise ObservabilityValidationError("integration asset failed_phase is unsupported") - for field_name, value in ( - ("target", self.target), - ("origin_relation", self.origin_relation), - ("staging_relation", self.staging_relation), - ("failed_phase", self.failed_phase), - ("loader", self.loader), - ): - validate_optional_identifier(value=value, field_name=field_name) - if self.rows_loaded is not None and ( - type(self.rows_loaded) is not int or self.rows_loaded < 0 - ): - raise ObservabilityValidationError("integration rows_loaded must be non-negative") - if self.materialized is not None and type(self.materialized) is not bool: - raise ObservabilityValidationError("integration materialized must be boolean") - validate_allowlisted_mapping( - value=self.future_cursor_safety, - field_name="future_cursor_safety", - allowed_keys=INTEGRATION_FUTURE_CURSOR_KEYS, - ) - validate_allowlisted_mapping( - value=self.microbatch, - field_name="microbatch", - allowed_keys=INTEGRATION_MICROBATCH_KEYS, - ) - validate_maximum_start_safety(self.maximum_start_safety) - for key in INTEGRATION_MICROBATCH_COUNT_KEYS: - count: object = self.microbatch.get(key) - if count is not None and (type(count) is not int or count < 0): - raise ObservabilityValidationError( - f"integration microbatch {key} must be non-negative integer" - ) - for value, field_name, allowed in ( - (self.microbatch.get("run_type"), "run_type", INTEGRATION_MICROBATCH_RUN_TYPES), - (self.microbatch.get("action"), "action", INTEGRATION_MICROBATCH_LIMIT_ACTIONS), - ( - self.microbatch.get("unaccounted_partition_policy"), - "unaccounted_partition_policy", - INTEGRATION_MICROBATCH_PARTITION_POLICIES, - ), - ( - self.microbatch.get("replay_requirement_state"), - "replay_requirement_state", - INTEGRATION_MICROBATCH_REPLAY_STATES, - ), - ): - if value is not None and (type(value) is not str or value not in allowed): - raise ObservabilityValidationError( - f"integration microbatch {field_name} is unsupported" - ) - concurrent_enabled: object = self.microbatch.get("concurrent_enabled") - if concurrent_enabled is not None and type(concurrent_enabled) is not bool: - raise ObservabilityValidationError( - "integration microbatch concurrent_enabled must be boolean" - ) - - -@dataclass(frozen=True) -class IntegrationCheckResult: - """Bounded check enrichment consumed by SQLBuild-owned integrations.""" - - kind: str - name: str - check_id: str - passed: bool - status: str - dag_check_id: str | None = None - severity: AuditSeverity | None = None - asset_name: str | None = None - attachment_kind: str | None = None - attached_column_name: str | None = None - attached_target_name: str | None = None - run_scope_phase: str | None = None - row_count: int | None = None - reused: bool | None = None - evaluation_mode: str | None = None - measured_value: float | None = None - sample_count: int | None = None - sample_unit: str | None = None - minimum_samples: int | None = None - thresholds: Mapping[str, object] | None = None - evidence_count: int | None = None - evidence_truncated: bool | None = None - audit_definition_name: str | None = None - audit_description: str | None = None - - def __post_init__(self) -> None: - for field_name, value in ( - ("check kind", self.kind), - ("check name", self.name), - ("check id", self.check_id), - ): - validate_identifier(value=value, field_name=field_name) - validate_optional_identifier(value=self.dag_check_id, field_name="dag_check_id") - if self.kind not in INTEGRATION_CHECK_KINDS: - raise ObservabilityValidationError("integration check kind is unsupported") - if self.kind == INTEGRATION_AUDIT_KIND and self.audit_definition_name is None: - raise ObservabilityValidationError( - "integration audit check requires audit_definition_name" - ) - if self.status not in INTEGRATION_CHECK_STATUSES: - raise ObservabilityValidationError("integration check status is unsupported") - if type(self.passed) is not bool: - raise ObservabilityValidationError("integration check passed must be boolean") - if self.passed != (self.status in INTEGRATION_CHECK_NON_FAIL_STATUSES): - raise ObservabilityValidationError("integration check status and passed are incoherent") - if self.severity is not None and self.severity not in INTEGRATION_CHECK_SEVERITIES: - raise ObservabilityValidationError("integration check severity is unsupported") - if ( - self.attachment_kind is not None - and self.attachment_kind not in INTEGRATION_CHECK_ATTACHMENT_KINDS - ): - raise ObservabilityValidationError("integration check attachment_kind is unsupported") - if ( - self.run_scope_phase is not None - and self.run_scope_phase not in INTEGRATION_CHECK_RUN_SCOPE_PHASES - ): - raise ObservabilityValidationError("integration check run_scope_phase is unsupported") - for field_name, value in ( - ("asset_name", self.asset_name), - ("attachment_kind", self.attachment_kind), - ("attached_column_name", self.attached_column_name), - ("attached_target_name", self.attached_target_name), - ("run_scope_phase", self.run_scope_phase), - ("audit_definition_name", self.audit_definition_name), - ): - validate_optional_identifier(value=value, field_name=field_name) - if self.row_count is not None and (type(self.row_count) is not int or self.row_count < 0): - raise ObservabilityValidationError("integration check row_count must be non-negative") - if self.reused is not None and type(self.reused) is not bool: - raise ObservabilityValidationError("integration check reused must be boolean") - if self.evaluation_mode is not None and self.evaluation_mode not in { - mode.value for mode in AuditEvaluationMode - }: - raise ObservabilityValidationError("integration check evaluation_mode is unsupported") - if self.measured_value is not None and ( - type(self.measured_value) is not float or not math.isfinite(self.measured_value) - ): - raise ObservabilityValidationError("integration check measured_value must be finite") - for field_name, value in ( - ("sample_count", self.sample_count), - ("minimum_samples", self.minimum_samples), - ("evidence_count", self.evidence_count), - ): - if value is not None and (type(value) is not int or value < 0): - raise ObservabilityValidationError( - f"integration check {field_name} must be non-negative" - ) - validate_optional_identifier(value=self.sample_unit, field_name="sample_unit") - if self.evidence_truncated is not None and type(self.evidence_truncated) is not bool: - raise ObservabilityValidationError( - "integration check evidence_truncated must be boolean" - ) - if self.audit_description is not None and ( - not isinstance(self.audit_description, str) or not self.audit_description.strip() - ): - raise ObservabilityValidationError( - "integration check audit_description must be a non-empty string" - ) - if self.thresholds is not None: - self._validate_measurement_thresholds() - - def _validate_measurement_thresholds(self) -> None: - thresholds: Mapping[str, object] = cast(Mapping[str, object], self.thresholds) - if not set(thresholds).issubset({"warn", "error"}) or not thresholds: - raise ObservabilityValidationError("integration check thresholds are invalid") - for bound in thresholds.values(): - if not isinstance(bound, Mapping): - raise ObservabilityValidationError("integration check threshold must be an object") - typed_bound: Mapping[str, object] = cast(Mapping[str, object], bound) - operator: object = typed_bound.get("operator") - expected_keys: set[str] - if operator in {ThresholdOperator.BELOW.value, ThresholdOperator.ABOVE.value}: - expected_keys = {"operator", "limit"} - elif operator == ThresholdOperator.OUTSIDE.value: - expected_keys = {"operator", "lower", "upper"} - else: - raise ObservabilityValidationError( - "integration check threshold operator is unsupported" - ) - if set(typed_bound) != expected_keys: - raise ObservabilityValidationError("integration check threshold bound is invalid") - numeric_values: tuple[object, ...] = tuple( - typed_bound[key] for key in expected_keys - {"operator"} - ) - if any( - type(value) is not float or not math.isfinite(cast(float, value)) - for value in numeric_values - ): - raise ObservabilityValidationError("integration check threshold bound is invalid") - _ = encode_integration_json(value=thresholds) - - -@dataclass(frozen=True) -class IntegrationResultEnvelope: - """Versioned canonical terminal and typed integration result envelope.""" - - schema_version: int - record_kind: str - event_id: str - event_sequence: int - event_type: str - occurred_at: str - invocation_id: str - run_id: str - resource_id: str - resource_attempt_id: str - operation_id: str | None - statement_id: str | None - resource_kind: str - resource_name: str - attempt_number: int - duration_ms: int | float | None - output_kind: IntegrationOutputKind - command: str - error_code: str | None = None - error_type: str | None = None - skip_code: str | None = None - skip_mode: str | None = None - projection_degraded: bool = False - asset: IntegrationAssetResult | None = None - checks: tuple[IntegrationCheckResult, ...] = () - - def __post_init__(self) -> None: - if ( - type(self.schema_version) is not int - or self.schema_version != INTEGRATION_RESULT_SCHEMA_VERSION - ): - raise ObservabilityValidationError( - f"unsupported integration result schema_version: {self.schema_version}" - ) - if self.record_kind != INTEGRATION_RESULT_RECORD_KIND: - raise ObservabilityValidationError( - f"unsupported integration result record_kind: {self.record_kind}" - ) - if self.event_type not in RESOURCE_TERMINALS: - raise ObservabilityValidationError( - f"integration result event_type is not a resource terminal: {self.event_type}" - ) - required_text: tuple[str, ...] = ( - self.event_id, - self.event_type, - self.occurred_at, - self.invocation_id, - self.run_id, - self.resource_id, - self.resource_attempt_id, - self.resource_kind, - self.resource_name, - self.command, - ) - if any(not isinstance(value, str) or not value for value in required_text): - raise ObservabilityValidationError( - "integration result identity fields must be non-empty" - ) - for field_name, value in ( - ("event_id", self.event_id), - ("invocation_id", self.invocation_id), - ("run_id", self.run_id), - ("resource_id", self.resource_id), - ("resource_attempt_id", self.resource_attempt_id), - ("resource_kind", self.resource_kind), - ("resource_name", self.resource_name), - ("command", self.command), - ): - validate_identifier(value=value, field_name=field_name) - if self.resource_kind not in INTEGRATION_RESOURCE_KINDS: - raise ObservabilityValidationError("integration resource_kind is unsupported") - validate_optional_identifier(value=self.operation_id, field_name="operation_id") - validate_optional_identifier(value=self.statement_id, field_name="statement_id") - try: - occurred_at: datetime = datetime.fromisoformat(self.occurred_at) - except ValueError as error: - raise ObservabilityValidationError( - "integration result occurred_at must be an ISO-8601 timestamp" - ) from error - if occurred_at.utcoffset() is None: - raise ObservabilityValidationError( - "integration result occurred_at must include a timezone" - ) - if type(self.event_sequence) is not int or self.event_sequence < 0: - raise ObservabilityValidationError( - "integration result event_sequence must be non-negative" - ) - if ( - not isinstance(self.attempt_number, int) - or isinstance(self.attempt_number, bool) - or self.attempt_number < 1 - ): - raise ObservabilityValidationError("integration result attempt_number must be positive") - if self.asset is None and not self.checks: - raise ObservabilityValidationError( - "integration result must contain asset or check enrichment" - ) - if self.duration_ms is not None and ( - type(self.duration_ms) not in (int, float) - or not math.isfinite(self.duration_ms) - or self.duration_ms < 0 - ): - raise ObservabilityValidationError( - "integration result duration_ms must be non-negative" - ) - for field_name, value in ( - ("error_code", self.error_code), - ("error_type", self.error_type), - ("skip_code", self.skip_code), - ("skip_mode", self.skip_mode), - ): - validate_optional_identifier(value=value, field_name=field_name) - if type(self.projection_degraded) is not bool: - raise ObservabilityValidationError( - "integration result projection_degraded must be boolean" - ) - self._validate_result_identity() - self._validate_terminal_facts() - - def to_json(self) -> str: - """Encode one validated envelope as a JSON Lines record.""" - - payload: dict[str, Any] = asdict(self) - payload["output_kind"] = self.output_kind.value - for check_payload, check in zip(payload["checks"], self.checks, strict=True): - if check.severity is not None: - check_payload["severity"] = check.severity.value - return encode_integration_json(value=payload, record=True) + "\n" - - def _validate_result_identity(self) -> None: - if self.output_kind == IntegrationOutputKind.ASSET: - if self.asset is None or self.checks: - raise ObservabilityValidationError("asset output must contain exactly one asset") - if self.asset.kind != self.resource_kind or self.asset.name != self.resource_name: - raise ObservabilityValidationError( - "integration asset identity does not match terminal" - ) - expected_id: str = integration_resource_id( - resource_kind=self.resource_kind, - resource_name=self.resource_name, - check_id=None, - loader_name=self.asset.loader, - ) - if self.resource_id != expected_id: - raise ObservabilityValidationError("integration asset resource_id is incoherent") - return - if self.output_kind == IntegrationOutputKind.CHECK: - if self.asset is not None or len(self.checks) != 1: - raise ObservabilityValidationError("check output must contain exactly one check") - if self.checks[0].check_id != self.resource_id: - raise ObservabilityValidationError( - "integration check identity does not match terminal" - ) - return - raise ObservabilityValidationError("integration result output_kind is unsupported") - - def _validate_terminal_facts(self) -> None: - if self.event_type == INTEGRATION_RESOURCE_FAILED_EVENT: - if self.error_type is None or self.skip_code is not None or self.skip_mode is not None: - raise ObservabilityValidationError("failed integration terminal facts are invalid") - if self.asset is not None and self.asset.status != INTEGRATION_FAILED_STATUS: - raise ObservabilityValidationError("failed terminal asset status is incoherent") - return - if self.event_type == RESOURCE_ATTEMPT_SKIPPED_EVENT: - if ( - self.skip_code not in RESOURCE_SKIP_CODES - or self.error_code is not None - or self.error_type is not None - or (self.skip_mode is not None and self.skip_mode not in INTEGRATION_SKIP_MODES) - ): - raise ObservabilityValidationError("skipped integration terminal facts are invalid") - if self.asset is not None and self.asset.status != INTEGRATION_SKIPPED_STATUS: - raise ObservabilityValidationError("skipped terminal asset status is incoherent") - return - if any( - value is not None - for value in (self.error_code, self.error_type, self.skip_code, self.skip_mode) - ): - raise ObservabilityValidationError("completed integration terminal has failure facts") - - @classmethod - def from_json(cls, value: str) -> IntegrationResultEnvelope: - """Decode and validate one integration result JSON record.""" - - if len(value.encode("utf-8")) > MAX_INTEGRATION_RECORD_BYTES: - raise ObservabilityValidationError("integration result record exceeds size limit") - try: - payload: Any = json.loads(value) - except json.JSONDecodeError as error: - raise ObservabilityValidationError("malformed integration result JSON") from error - if not isinstance(payload, dict): - raise ObservabilityValidationError("integration result must be a JSON object") - if ( - type(payload.get("schema_version")) is not int - or payload.get("schema_version") != INTEGRATION_RESULT_SCHEMA_VERSION - ): - raise ObservabilityValidationError( - f"unsupported integration result schema_version: {payload.get('schema_version')}" - ) - if payload.get("record_kind") != INTEGRATION_RESULT_RECORD_KIND: - raise ObservabilityValidationError( - f"unsupported integration result record_kind: {payload.get('record_kind')}" - ) - asset_payload: object = payload.pop("asset", None) - checks_payload: object = payload.pop("checks", ()) - if asset_payload is not None and not isinstance(asset_payload, dict): - raise ObservabilityValidationError("integration result asset must be an object") - if not isinstance(checks_payload, list | tuple): - raise ObservabilityValidationError("integration result checks must be an array") - if len(checks_payload) > MAX_INTEGRATION_COLLECTION_ITEMS: - raise ObservabilityValidationError("integration result checks exceed item limit") - try: - output_kind: IntegrationOutputKind = IntegrationOutputKind(payload.pop("output_kind")) - asset: IntegrationAssetResult | None = ( - IntegrationAssetResult(**asset_payload) if isinstance(asset_payload, dict) else None - ) - parsed_checks: list[IntegrationCheckResult] = [] - for check_payload in checks_payload: - if not isinstance(check_payload, dict): - continue - severity_value: object = check_payload.pop("severity", None) - severity: AuditSeverity | None = None - if severity_value is not None: - try: - severity = AuditSeverity(severity_value) - except (TypeError, ValueError) as error: - allowed: str = ", ".join(item.value for item in AuditSeverity) - raise ObservabilityValidationError( - f"integration check severity must be one of: {allowed}" - ) from error - parsed_checks.append(IntegrationCheckResult(**check_payload, severity=severity)) - checks: tuple[IntegrationCheckResult, ...] = tuple(parsed_checks) - if len(checks) != len(checks_payload): - raise ObservabilityValidationError( - "integration result checks must contain only objects" - ) - return cls( - **payload, - output_kind=output_kind, - asset=asset, - checks=checks, - ) - except ObservabilityValidationError: - raise - except (KeyError, TypeError, ValueError) as error: - raise ObservabilityValidationError("invalid integration result envelope") from error +from sqlbuild.cli.output.types import WrittenTargetDiagnostic @dataclass(frozen=True) @@ -552,23 +21,6 @@ class WrittenTarget: diagnostics: tuple[WrittenTargetDiagnostic, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class SkillInstallTarget: - """One destination for a SQLBuild skill file.""" - - name: str - path: Path - - -@dataclass(frozen=True) -class SkillSettings: - """Project skill installation preferences.""" - - targets: tuple[str, ...] - auto_update: bool - configured: bool - - @dataclass(frozen=True) class SkillMaintenanceResult: """Non-blocking generated-skill maintenance result.""" diff --git a/src/sqlbuild/cli/output/types.py b/src/sqlbuild/cli/output/types.py index 85785c3f17..753c8e8c0a 100644 --- a/src/sqlbuild/cli/output/types.py +++ b/src/sqlbuild/cli/output/types.py @@ -1,16 +1,8 @@ """Type-layer declarations for CLI output.""" -from enum import StrEnum from typing import TYPE_CHECKING if TYPE_CHECKING: from sqlbuild.compiler.compile.models import CompilerDiagnostic type WrittenTargetDiagnostic = CompilerDiagnostic - - -class IntegrationOutputKind(StrEnum): - """Kind of integration-facing result enrichment.""" - - ASSET = "asset" - CHECK = "check" diff --git a/src/sqlbuild/cli/progress/classes/audit_progress_reporter.py b/src/sqlbuild/cli/progress/classes/audit_progress_reporter.py index ba59b10e59..b6f2bc1551 100644 --- a/src/sqlbuild/cli/progress/classes/audit_progress_reporter.py +++ b/src/sqlbuild/cli/progress/classes/audit_progress_reporter.py @@ -6,7 +6,7 @@ from collections.abc import Callable from typing import TextIO -from sqlbuild.cli.progress.classes.native_progress_projector import ( +from sqlbuild._native import ( NativeProgressProjector, current_native_progress_projector, ) diff --git a/src/sqlbuild/cli/progress/classes/connection_progress_reporter.py b/src/sqlbuild/cli/progress/classes/connection_progress_reporter.py deleted file mode 100644 index 57253b2757..0000000000 --- a/src/sqlbuild/cli/progress/classes/connection_progress_reporter.py +++ /dev/null @@ -1,65 +0,0 @@ -"""User-facing warehouse connection progress reporter.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.phase_line import format_phase_line -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator - - -class ConnectionProgressReporter: - """Render simple connection start, success, and failure messages.""" - - def __init__( - self, - *, - adapter_name: str, - stream: TextIO, - blank_line_before_start: bool = False, - blank_line_after_complete: bool = False, - use_color: bool = False, - ) -> None: - self._adapter_name: str = adapter_name - self._stream: TextIO = stream - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._blank_line_before_start: bool = blank_line_before_start - self._blank_line_after_complete: bool = blank_line_after_complete - self._style: CliStyle = CliStyle(use_color=use_color) - - def on_connection_start(self, connection_count: int) -> None: - if self._blank_line_before_start: - self._write("") - self._write(self._style.muted(self._start_message(connection_count))) - - def on_connection_complete(self, *, connection_count: int, elapsed_seconds: float) -> None: - del connection_count - message: str = format_phase_line( - style=self._style, - ok=True, - label="Warehouse connected", - summary=f"{self._adapter_name} ({elapsed_seconds:.2f}s)", - ) - self._write(message) - if self._blank_line_after_complete: - self._write("") - - def on_connection_error(self, *, connection_count: int, elapsed_seconds: float) -> None: - del connection_count - message: str = format_phase_line( - style=self._style, - ok=False, - label="Warehouse connection failed", - summary=f"{self._adapter_name} (after {elapsed_seconds:.2f}s)", - ) - self._write(message) - - def _start_message(self, connection_count: int) -> str: - if connection_count <= 1: - return f"Connecting to {self._adapter_name}..." - return f"Connecting to {self._adapter_name} ({connection_count} connections)..." - - def _write(self, message: str) -> None: - self._lines.write_persistent(stream=self._stream, text=f"{message}\n") diff --git a/src/sqlbuild/cli/progress/classes/native_progress_projector.py b/src/sqlbuild/cli/progress/classes/native_progress_projector.py deleted file mode 100644 index 78d8f57f9e..0000000000 --- a/src/sqlbuild/cli/progress/classes/native_progress_projector.py +++ /dev/null @@ -1,574 +0,0 @@ -"""Invocation-scoped native CLI projection of canonical lifecycle facts.""" - -from __future__ import annotations - -import threading -from collections import defaultdict, deque -from collections.abc import Mapping -from contextvars import ContextVar, Token -from typing import TextIO, cast - -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator -from sqlbuild.runtime.observability.constants import ( - RESOURCE_TERMINALS, - RETRY_SCHEDULED_EVENT, - STATEMENT_FAILED_EVENT, -) -from sqlbuild.runtime.observability.models import LifecycleEvent - -_VISIBLE_OPERATION_LABELS: Mapping[str, str] = { - "clone_finalization": "Clone finalization", - "discovery_declaration_parse": "Declaration parsing", - "discovery_filesystem_walk": "Filesystem discovery", - "discovery_project_assembly": "Project assembly", - "discovery_python_import": "Python discovery", - "external_manifest_discovery": "External manifest discovery", - "project_discovery": "Project discovery", - "project_compile": "Project compile", - "sql_test_planning": "SQL test planning", - "dbt_command": "dbt command", - "external_source_load": "loader callable", - "ingestr_command": "ingestr subprocess", - "managed_source_load": "loader callable", - "python_asset": "Python asset attempt", - "python_check": "Python check attempt", - "python_hook": "Python hook", - "python_task": "Python task attempt", - "sql_hook": "SQL hook", - "audit_evaluation": "Audit evaluation", - "sql_test_setup": "SQL test setup", - "sql_test_assertion": "SQL test assertion", - "source_freshness_metadata_observation": "Freshness metadata", - "source_freshness_query_observation": "Freshness query", - "source_freshness_publication": "Publish freshness state", - "retention_inspection": "Inspect retention", - "retention_application": "Apply retention", - "table_type_inspection": "Inspect table type", - "table_type_conversion": "Convert table type", - "schema_synchronization": "Synchronize schema", - "staging_creation": "Create staging relation", - "relation_promotion": "Promote relation", -} -_TTY_TRANSIENT_OWNER_OPERATIONS: frozenset[str] = frozenset( - { - "dbt_command", - "discovery_declaration_parse", - "discovery_filesystem_walk", - "discovery_project_assembly", - "discovery_python_import", - "external_manifest_discovery", - "external_source_load", - "ingestr_command", - "managed_source_load", - "project_compile", - "project_discovery", - "sql_test_planning", - "python_asset", - "python_check", - "python_hook", - "python_task", - "sql_hook", - "sql_test_setup", - } -) -_TTY_RESOURCE_OWNER_OPERATIONS: frozenset[str] = frozenset( - {"audit_evaluation", "sql_test_assertion"} -) -_FRESHNESS_METADATA_OPERATION: str = "source_freshness_metadata_observation" -_FRESHNESS_QUERY_OPERATION: str = "source_freshness_query_observation" -_RESOURCE_START: str = "resource_attempt_started" -_SCENARIO_RESOURCE_PREFIX: str = "scenario:" -_OPERATION_START: str = "operation_started" -_OPERATION_TERMINALS: frozenset[str] = frozenset({"operation_completed", "operation_failed"}) -_STATEMENT_START: str = "statement_started" -_STATEMENT_SUBMITTED: str = "statement_submitted" -_STATEMENT_HEARTBEAT: str = "statement_heartbeat" -_STATEMENT_TERMINALS: frozenset[str] = frozenset({"statement_completed", "statement_failed"}) - - -class NativeProgressProjector: - """Project canonical starts and correlate enriched terminal presentation.""" - - def __init__(self, *, stream: TextIO, use_color: bool, debug: bool = False) -> None: - self._stream: TextIO = stream - self._debug: bool = debug - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._style: CliStyle = CliStyle(use_color=use_color) - self._is_tty: bool = hasattr(stream, "isatty") and stream.isatty() - self._lock: threading.RLock = self._lines.lock - self._seen_event_ids: set[str] = set() - self._resource_ordinals: dict[str, int] = {} - self._resource_total: int = 0 - self._attempts_by_id: dict[str, LifecycleEvent] = {} - self._pending_terminal_ids_by_resource: defaultdict[str, deque[str]] = defaultdict(deque) - self._resource_ids_by_name: defaultdict[str, set[str]] = defaultdict(set) - self._rendered_terminal_ids: set[str] = set() - self._operation_starts: dict[str, LifecycleEvent] = {} - self._quiet_operation_ids: set[str] = set() - self._enriched_resource_names: set[str] = set() - self._resource_names: dict[str, str] = {} - self._statement_starts: dict[str, LifecycleEvent] = {} - self._statement_query_ids: dict[str, str] = {} - - def install(self) -> Token[NativeProgressProjector | None]: - """Install this projector for presentation adapters in the invocation context.""" - - return _CURRENT_PROJECTOR.set(self) - - def restore(self, token: Token[NativeProgressProjector | None]) -> None: - """Restore the previous invocation projector.""" - - _CURRENT_PROJECTOR.reset(token) - - def configure_resources(self, *, ordinals: Mapping[str, int], total: int) -> None: - """Add safe display order metadata without changing canonical facts.""" - - with self._lock: - self._resource_ordinals.update(ordinals) - self._resource_total = max(self._resource_total, total) - self._enriched_resource_names.update(ordinals) - - def expect_resource_enrichment(self, *, resource_name: str) -> None: - """Declare that an existing result callback will enrich this resource terminal.""" - - with self._lock: - self._enriched_resource_names.add(resource_name) - - def is_operation_active(self, *, operation_name: str) -> bool: - """Return whether a visible canonical operation currently owns progress.""" - - with self._lock: - return any( - event.payload.get("operation_name") == operation_name - for event in self._operation_starts.values() - ) - - def consume(self, event: LifecycleEvent) -> None: - """Consume one known lifecycle fact idempotently.""" - - with self._lock: - if event.event_id in self._seen_event_ids: - return - self._seen_event_ids.add(event.event_id) - if event.event_type == _RESOURCE_START: - self._consume_resource_start(event) - elif event.event_type in RESOURCE_TERMINALS: - self._consume_resource_terminal(event) - elif event.event_type == _OPERATION_START: - self._consume_operation_start(event) - elif event.event_type in _OPERATION_TERMINALS: - self._consume_operation_terminal(event) - elif event.event_type == RETRY_SCHEDULED_EVENT: - self._consume_retry_scheduled(event) - elif event.event_type == _STATEMENT_START: - self._consume_statement_start(event) - elif event.event_type == _STATEMENT_SUBMITTED: - self._consume_statement_submitted(event) - elif event.event_type == _STATEMENT_HEARTBEAT: - self._consume_statement_heartbeat(event) - elif event.event_type in _STATEMENT_TERMINALS: - self._consume_statement_terminal(event) - - def consume_resource_terminal( - self, - *, - resource_name: str, - resource_id: str | None = None, - resource_attempt_id: str | None = None, - attempt_number: int | None = None, - ) -> float | None: - """Claim the targeted or latest terminal for one final resource result.""" - - with self._lock: - resolved_resource_id, pending, candidates, target = self._resolve_resource_terminal( - resource_name=resource_name, - resource_id=resource_id, - resource_attempt_id=resource_attempt_id, - attempt_number=attempt_number, - ) - if resolved_resource_id is None or pending is None or target is None: - return None - target_attempt_id: str | None = target.resource_attempt_id - if target_attempt_id is None: - return None - for event in candidates: - if event is target: - break - self._render_resource_terminal(event=event, resource_name=resource_name) - self._rendered_terminal_ids.add(target_attempt_id) - self._remove_claimed_terminals(pending=pending, target=target) - duration: object = target.payload.get("duration_ms") - return float(duration) if isinstance(duration, int | float) else None - - def resource_terminal_duration( - self, - *, - resource_name: str, - resource_id: str | None = None, - resource_attempt_id: str | None = None, - attempt_number: int | None = None, - ) -> float | None: - """Return a terminal duration without claiming its presentation.""" - - with self._lock: - _, _, _, target = self._resolve_resource_terminal( - resource_name=resource_name, - resource_id=resource_id, - resource_attempt_id=resource_attempt_id, - attempt_number=attempt_number, - ) - if target is None: - return None - duration: object = target.payload.get("duration_ms") - return float(duration) if isinstance(duration, int | float) else None - - def close(self) -> None: - """Restore terminal cursor state without inventing missing terminal rows.""" - - with self._lock: - event: LifecycleEvent - for event in self._attempts_by_id.values(): - if event.event_type not in RESOURCE_TERMINALS: - continue - resource_name: object = event.payload.get("resource_name") - if isinstance(resource_name, str): - self._render_resource_terminal(event=event, resource_name=resource_name) - - def _consume_resource_start(self, event: LifecycleEvent) -> None: - attempt_id: str | None = event.resource_attempt_id - resource_id: str | None = event.resource_id - resource_name: object = event.payload.get("resource_name") - resource_kind: object = event.payload.get("resource_kind") - if ( - attempt_id is None - or resource_id is None - or not isinstance(resource_name, str) - or not isinstance(resource_kind, str) - ): - return - self._resource_ids_by_name[resource_name].add(resource_id) - self._resource_names[resource_id] = resource_name - self._render_prior_attempts(resource_id=resource_id, resource_name=resource_name) - self._attempts_by_id[attempt_id] = event - if self._is_tty and (resource_name in self._enriched_resource_names or not self._debug): - return - ordinal: int | None = self._resource_ordinals.get(resource_name) - counter: str = "" - if ordinal is not None and self._resource_total > 0: - counter = f"{ordinal}/{self._resource_total} " - status: str = self._style.status(status="START") - display_name: str = _display_resource_name( - resource_id=resource_id, resource_name=resource_name - ) - self._write(f" {counter}{resource_kind:<10}{display_name} {status}") - - def _consume_resource_terminal(self, event: LifecycleEvent) -> None: - attempt_id: str | None = event.resource_attempt_id - resource_id: str | None = event.resource_id - resource_name: object = event.payload.get("resource_name") - if attempt_id is None or resource_id is None or not isinstance(resource_name, str): - return - self._resource_ids_by_name[resource_name].add(resource_id) - self._attempts_by_id[attempt_id] = event - self._pending_terminal_ids_by_resource[resource_id].append(attempt_id) - if resource_name not in self._enriched_resource_names: - self._render_resource_terminal(event=event, resource_name=resource_name) - - def _render_resource_terminal(self, *, event: LifecycleEvent, resource_name: str) -> None: - attempt_id: str | None = event.resource_attempt_id - if attempt_id is None or attempt_id in self._rendered_terminal_ids: - return - self._rendered_terminal_ids.add(attempt_id) - resource_kind: object = event.payload.get("resource_kind") - if not isinstance(resource_kind, str): - return - status_text: str = ( - "FAIL" - if event.event_type.endswith("failed") - else "SKIP" - if event.event_type.endswith("skipped") - else "OK" - ) - status: str = self._style.status(status=status_text) - duration: object = event.payload.get("duration_ms") - elapsed: str = ( - f" {float(duration) / 1000.0:.2f}s" if isinstance(duration, int | float) else "" - ) - display_name: str = _display_resource_name( - resource_id=event.resource_id, resource_name=resource_name - ) - self._write(f" {resource_kind:<10}{display_name} {status}{elapsed}") - - def _render_prior_attempts(self, *, resource_id: str, resource_name: str) -> None: - pending: deque[str] = self._pending_terminal_ids_by_resource[resource_id] - while pending: - attempt_id: str = pending.popleft() - event: LifecycleEvent | None = self._attempts_by_id.get(attempt_id) - if event is not None: - self._render_resource_terminal(event=event, resource_name=resource_name) - - def _pending_terminal_events(self, *, pending: deque[str]) -> list[LifecycleEvent]: - events: list[LifecycleEvent] = [] - for attempt_id in pending: - if attempt_id in self._rendered_terminal_ids: - continue - event: LifecycleEvent | None = self._attempts_by_id.get(attempt_id) - if event is not None: - events.append(event) - return events - - def _remove_claimed_terminals(self, *, pending: deque[str], target: LifecycleEvent) -> None: - while pending: - attempt_id: str = pending.popleft() - if attempt_id == target.resource_attempt_id: - return - - def _single_resource_id(self, *, resource_name: str) -> str | None: - resource_ids: set[str] = self._resource_ids_by_name[resource_name] - if len(resource_ids) != 1: - return None - return next(iter(resource_ids)) - - def _resolve_resource_terminal( - self, - *, - resource_name: str, - resource_id: str | None, - resource_attempt_id: str | None, - attempt_number: int | None, - ) -> tuple[str | None, deque[str] | None, list[LifecycleEvent], LifecycleEvent | None]: - resolved_resource_id: str | None = resource_id or self._single_resource_id( - resource_name=resource_name - ) - if resolved_resource_id is None: - return None, None, [], None - pending: deque[str] = self._pending_terminal_ids_by_resource[resolved_resource_id] - candidates: list[LifecycleEvent] = self._pending_terminal_events(pending=pending) - target: LifecycleEvent | None = _claim_target( - candidates=candidates, - resource_attempt_id=resource_attempt_id, - attempt_number=attempt_number, - ) - return resolved_resource_id, pending, candidates, target - - def _consume_operation_start(self, event: LifecycleEvent) -> None: - operation_id: str | None = event.operation_id - operation_name: object = event.payload.get("operation_name") - if ( - operation_id is None - or not isinstance(operation_name, str) - or operation_name not in _VISIBLE_OPERATION_LABELS - or ( - self._is_tty - and ( - ( - event.resource_attempt_id is None - and operation_name in _TTY_TRANSIENT_OWNER_OPERATIONS - ) - or ( - event.resource_attempt_id is not None - and operation_name in _TTY_RESOURCE_OWNER_OPERATIONS - ) - ) - ) - ): - return - self._operation_starts[operation_id] = event - if event.resource_attempt_id is not None and not self._debug: - self._quiet_operation_ids.add(operation_id) - return - status: str = self._style.status(status="START") - prefix: str = " " if event.resource_attempt_id is not None else "" - attempt: str = _operation_attempt_label(event) - self._write(f"{prefix}{_VISIBLE_OPERATION_LABELS[operation_name]}{attempt} {status}") - - def _consume_operation_terminal(self, event: LifecycleEvent) -> None: - operation_id: str | None = event.operation_id - if operation_id is None or operation_id not in self._operation_starts: - return - start: LifecycleEvent = self._operation_starts.pop(operation_id) - operation_name: object = start.payload.get("operation_name") - failed: bool = event.event_type.endswith("failed") - if operation_id in self._quiet_operation_ids: - self._quiet_operation_ids.discard(operation_id) - if not failed: - return - if not isinstance(operation_name, str): - return - status_text: str = "FAIL" if failed else "OK" - status: str = self._style.status(status=status_text) - duration: object = event.payload.get("duration_ms") - elapsed: str = ( - f" ({float(duration) / 1000.0:.2f}s)" if isinstance(duration, int | float) else "" - ) - prefix: str = " " if event.resource_attempt_id is not None else "" - attempt: str = _operation_attempt_label(start) - detail: str = _operation_terminal_detail(operation_name=operation_name, event=event) - self._write( - f"{prefix}{_VISIBLE_OPERATION_LABELS[operation_name]}{attempt} " - f"{status}{detail}{elapsed}" - ) - - def _consume_retry_scheduled(self, event: LifecycleEvent) -> None: - failed_attempt: object = event.payload.get("failed_attempt_number") - next_attempt: object = event.payload.get("next_attempt_number") - delay_ms: object = event.payload.get("delay_ms") - if isinstance(failed_attempt, bool) or not isinstance(failed_attempt, int): - return - if isinstance(next_attempt, bool) or not isinstance(next_attempt, int): - return - if isinstance(delay_ms, bool) or not isinstance(delay_ms, int): - return - self._write( - f" retry {failed_attempt}->{next_attempt} in {float(delay_ms) / 1000.0:.2f}s" - ) - - def _consume_statement_start(self, event: LifecycleEvent) -> None: - statement_id: str | None = event.statement_id - if statement_id is None or event.resource_id not in self._resource_names: - return - self._statement_starts[statement_id] = event - if self._debug: - self._write(f" statement {self._statement_context(event)} START") - - def _consume_statement_submitted(self, event: LifecycleEvent) -> None: - statement_id: str | None = event.statement_id - query_id: object = event.payload.get("query_id") - if ( - statement_id is None - or statement_id not in self._statement_starts - or not isinstance(query_id, str) - ): - return - if self._statement_query_ids.get(statement_id) == query_id: - return - self._statement_query_ids[statement_id] = query_id - if not self._debug: - return - start: LifecycleEvent = self._statement_starts.get(statement_id, event) - self._write(f" statement {self._statement_context(start)} query_id={query_id}") - - def _consume_statement_heartbeat(self, event: LifecycleEvent) -> None: - if event.statement_id not in self._statement_starts: - return - duration: object = event.payload.get("duration_ms") - if not isinstance(duration, int | float): - return - query_id: str | None = self._statement_query_id(event) - query_suffix: str = f" query_id={query_id}" if query_id is not None else "" - self._write( - f" statement {self._statement_context(event)} " - f"RUNNING {float(duration) / 1000.0:.1f}s{query_suffix}" - ) - - def _consume_statement_terminal(self, event: LifecycleEvent) -> None: - statement_id: str | None = event.statement_id - if statement_id is None or statement_id not in self._statement_starts: - return - start: LifecycleEvent = self._statement_starts.pop(statement_id, event) - query_id: str | None = self._statement_query_id(event) - if event.event_type == STATEMENT_FAILED_EVENT: - query_suffix: str = f" query_id={query_id}" if query_id is not None else "" - self._write(f" statement {self._statement_context(start)} FAIL{query_suffix}") - self._statement_query_ids.pop(statement_id, None) - - def _statement_query_id(self, event: LifecycleEvent) -> str | None: - query_id: object = event.payload.get("query_id") - if isinstance(query_id, str): - return query_id - statement_id: str | None = event.statement_id - return None if statement_id is None else self._statement_query_ids.get(statement_id) - - def _statement_context(self, event: LifecycleEvent) -> str: - resource_name: str = self._resource_names.get(event.resource_id or "", "warehouse") - operation: LifecycleEvent | None = self._operation_starts.get(event.operation_id or "") - phase: object | None = None if operation is None else operation.payload.get("phase") - if not isinstance(phase, str) and operation is not None: - phase = operation.payload.get("operation_name") - if not isinstance(phase, str): - phase = event.payload.get("intent", "execute") - statement_kind: object = event.payload.get("statement_kind", "UNKNOWN") - return f"model={resource_name} phase={phase} kind={statement_kind}" - - def _write(self, line: str) -> None: - self._lines.write_persistent(stream=self._stream, text=f"{line}\n") - - -_CURRENT_PROJECTOR: ContextVar[NativeProgressProjector | None] = ContextVar( - "sqlbuild_native_progress_projector", default=None -) - - -def _display_resource_name(*, resource_id: str | None, resource_name: str) -> str: - if resource_id is None or not resource_id.startswith(_SCENARIO_RESOURCE_PREFIX): - return resource_name - scenario_name: str = resource_id.removeprefix(_SCENARIO_RESOURCE_PREFIX).split(":", 1)[0] - return f"{scenario_name}/{resource_name}" - - -def current_native_progress_projector() -> NativeProgressProjector | None: - """Return the projector installed for the current CLI invocation.""" - - return _CURRENT_PROJECTOR.get() - - -def _claim_target( - *, - candidates: list[LifecycleEvent], - resource_attempt_id: str | None, - attempt_number: int | None, -) -> LifecycleEvent | None: - if resource_attempt_id is not None: - return next( - (event for event in candidates if event.resource_attempt_id == resource_attempt_id), - None, - ) - if attempt_number is not None: - return next( - ( - event - for event in reversed(candidates) - if event.payload.get("attempt_number") == attempt_number - ), - None, - ) - return candidates[-1] if candidates else None - - -def _operation_attempt_label(event: LifecycleEvent) -> str: - metadata: object = event.payload.get("metadata") - if not isinstance(metadata, Mapping): - return "" - typed_metadata: Mapping[str, object] = cast(Mapping[str, object], metadata) - attempt_number: object = typed_metadata.get("attempt_number") - if isinstance(attempt_number, int) and not isinstance(attempt_number, bool): - return f" {attempt_number}" - return "" - - -def _operation_terminal_detail(*, operation_name: str, event: LifecycleEvent) -> str: - if operation_name == _FRESHNESS_QUERY_OPERATION: - return " (source freshness unknown)" if event.event_type.endswith("failed") else "" - if operation_name != _FRESHNESS_METADATA_OPERATION: - return "" - metadata: object = event.payload.get("metadata") - typed_metadata: Mapping[str, object] = ( - cast(Mapping[str, object], metadata) if isinstance(metadata, Mapping) else {} - ) - item_count: object = typed_metadata.get("item_count") - if isinstance(item_count, bool) or not isinstance(item_count, int): - return "" - noun: str = "source" if item_count == 1 else "sources" - if event.event_type.endswith("failed"): - error_type: object = event.payload.get("error_type") - reason: str = f": {error_type}" if isinstance(error_type, str) else "" - return f" ({item_count} {noun} unknown, metadata lookup failed{reason})" - unknown_count: object = typed_metadata.get("unknown_count") - unknown: int = ( - unknown_count - if isinstance(unknown_count, int) and not isinstance(unknown_count, bool) - else 0 - ) - return f" ({item_count} {noun}, {unknown} unknown)" diff --git a/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py b/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py index e39285e72b..8b6c6ec396 100644 --- a/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py +++ b/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py @@ -5,7 +5,7 @@ import threading from typing import TextIO -from sqlbuild.cli.progress.classes.native_progress_projector import ( +from sqlbuild._native import ( NativeProgressProjector, current_native_progress_projector, ) diff --git a/src/sqlbuild/cli/progress/classes/planning_progress_reporter.py b/src/sqlbuild/cli/progress/classes/planning_progress_reporter.py deleted file mode 100644 index 7dbe0d24d6..0000000000 --- a/src/sqlbuild/cli/progress/classes/planning_progress_reporter.py +++ /dev/null @@ -1,91 +0,0 @@ -"""User-facing plan generation progress reporter.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.cli.progress.classes.native_progress_projector import ( - NativeProgressProjector, - current_native_progress_projector, -) -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter - -_PROJECT_COMPILE_OPERATION: str = "project_compile" - - -class PlanningProgressReporter: - """Render simple planning phase progress messages.""" - - def __init__(self, *, stream: TextIO, use_color: bool = False) -> None: - self._status: TransientStatusReporter = TransientStatusReporter( - stream=stream, - use_color=use_color, - ) - self._active: bool = False - self._projector: NativeProgressProjector | None = current_native_progress_projector() - - def on_progress(self, message: str) -> None: - if self._projector is not None and self._projector.is_operation_active( - operation_name=_PROJECT_COMPILE_OPERATION - ): - return - if _is_planning_completion_message(message): - self.complete(message) - return - if not self._active: - self.start(message) - return - self.update(message) - - def start(self, message: str) -> None: - self._status.start(message) - self._active = True - - def update(self, message: str) -> None: - if not self._active: - self.start(message) - return - self._status.update(message) - - def complete(self, message: str) -> None: - self._status.complete(message=message) - self._active = False - - def complete_styled(self, message: str) -> None: - """Complete with a pre-styled message written verbatim.""" - - self._status.complete_styled(message=message) - self._active = False - - def error(self, message: str) -> None: - """Close active progress and render a non-success terminal message.""" - - self._status.error(message) - self._active = False - - def finish(self, *, blank_line_after: bool = False) -> None: - self._status.close() - self._active = False - if blank_line_after: - self._status.write_blank_line() - - -def _is_planning_completion_message(message: str) -> bool: - return message.startswith( - ( - "Built ", - "Checked ", - "Compiled ", - "Evaluated ", - "Applied ", - "Finalized ", - "Generated ", - "Inspected ", - "Loaded ", - "Planned ", - "Recorded ", - "Refreshed ", - "Resolved ", - "Cloned ", - ) - ) diff --git a/src/sqlbuild/cli/progress/classes/verbose_inspection_writer.py b/src/sqlbuild/cli/progress/classes/verbose_inspection_writer.py deleted file mode 100644 index 67409dd694..0000000000 --- a/src/sqlbuild/cli/progress/classes/verbose_inspection_writer.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Verbose per-query warehouse inspection lines written to stderr.""" - -from __future__ import annotations - -import sys -import threading - -from sqlbuild.adapter.relations.models import InspectionQueryRecord -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator - -_SQL_PREVIEW_LENGTH: int = 160 - - -class VerboseInspectionWriter: - """Write each inspection read as one truncated stderr line and keep a running total.""" - - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._count: int = 0 - self._seconds: float = 0.0 - - def write(self, record: InspectionQueryRecord) -> None: - """Write one inspection read; safe to call from inspection worker threads.""" - - with self._lock: - self._count += 1 - self._seconds += record.elapsed_seconds - self._write_line(text=self.format_record(record=record)) - - def write_total(self) -> None: - """Write the read count and cumulative query time when anything was read.""" - - if self._count: - self._write_line( - text=( - f"Inspection queries: {self._count} " - f"({self._seconds:.2f}s cumulative query time)" - ) - ) - - @staticmethod - def format_record(*, record: InspectionQueryRecord) -> str: - """Render one inspection read as a single truncated diagnostic line.""" - - text: str = " ".join(record.sql.split()) - if len(text) > _SQL_PREVIEW_LENGTH: - text = text[: _SQL_PREVIEW_LENGTH - 3] + "..." - outcome: str = ( - f"failed: {' '.join(record.error.split())}" - if record.error is not None - else _row_count_text(record.row_count) - ) - return f" inspect {record.elapsed_seconds:6.2f}s {outcome} {text}" - - @staticmethod - def _write_line(*, text: str) -> None: - shared_transient_line_coordinator().write_persistent(stream=sys.stderr, text=f"{text}\n") - - -def _row_count_text(row_count: int | None) -> str: - if row_count is None: - return "? rows" - return f"{row_count} row" if row_count == 1 else f"{row_count} rows" diff --git a/src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py b/src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py index 278dd806da..826778e69f 100644 --- a/src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py +++ b/src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py @@ -4,8 +4,7 @@ from typing import TextIO -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter +from sqlbuild._native import ConnectionProgressReporter, PlanningProgressReporter from sqlbuild.cli.progress.models import CommandProgressReporters diff --git a/src/sqlbuild/cli/progress/main/_execution_header.py b/src/sqlbuild/cli/progress/main/_execution_header.py deleted file mode 100644 index 52b73231b2..0000000000 --- a/src/sqlbuild/cli/progress/main/_execution_header.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Generic CLI execution header formatting.""" - -from __future__ import annotations - - -def format_execution_header(*, command: str, target: str | None, concurrency: int) -> str: - """Format command and execution context for a progress header.""" - - parts: list[str] = [command] - context_parts: list[str] = [] - if target is not None: - context_parts.append(f"target: {target}") - context_parts.append(f"concurrency: {concurrency}") - if context_parts: - parts.append(f" ({', '.join(context_parts)})") - return "".join(parts) diff --git a/src/sqlbuild/cli/progress/main/_expectation_detail.py b/src/sqlbuild/cli/progress/main/_expectation_detail.py deleted file mode 100644 index 910adc22fc..0000000000 --- a/src/sqlbuild/cli/progress/main/_expectation_detail.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Neutral SQL expectation detail formatting.""" - -from __future__ import annotations - -from sqlbuild.executor.testing.models import SqlTestDifferenceSample, StepResult -from sqlbuild.executor.testing.types import SqlTestOutcome - - -def format_expectation_detail(step_result: StepResult) -> str: - """Format failing row count detail for an expectation row.""" - - if step_result.outcome == SqlTestOutcome.PASS: - return "" - if step_result.outcome == SqlTestOutcome.ERROR: - return "" - if step_result.model_name.startswith("assertion "): - row_label: str = "row" if step_result.actual_row_count == 1 else "rows" - return f" {step_result.actual_row_count} {row_label}" - if ( - step_result.unexpected_row_count is None - and step_result.missing_row_count is None - and not step_result.unexpected_samples - and not step_result.missing_samples - ): - return f" {step_result.mismatched_row_count} mismatched" - parts: list[str] = [ - f"unexpected={step_result.unexpected_row_count}", - f"missing={step_result.missing_row_count}", - ] - if step_result.actual_row_count != step_result.expected_row_count: - parts.append( - f"row counts actual={step_result.actual_row_count} " - f"expected={step_result.expected_row_count}" - ) - parts.extend(_format_samples(label="unexpected sample", samples=step_result.unexpected_samples)) - parts.extend(_format_samples(label="missing sample", samples=step_result.missing_samples)) - for difference in step_result.column_differences: - parts.append( - f"differing column {difference.name}: actual={difference.actual}, " - f"expected={difference.expected}" - ) - return f" {'; '.join(parts)}" - - -def _format_samples(*, label: str, samples: tuple[SqlTestDifferenceSample, ...]) -> tuple[str, ...]: - """Format already-safe representative rows for terminal output.""" - - lines: list[str] = [] - for index, sample in enumerate(samples, start=1): - rendered_values: str = ", ".join(f"{name}={value}" for name, value in sample.values) - lines.append(f"{label} {index}: {rendered_values}") - return tuple(lines) diff --git a/src/sqlbuild/cli/progress/main/_expectation_name.py b/src/sqlbuild/cli/progress/main/_expectation_name.py deleted file mode 100644 index 8ad4a46191..0000000000 --- a/src/sqlbuild/cli/progress/main/_expectation_name.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Neutral SQL expectation name formatting.""" - -from __future__ import annotations - - -def format_expectation_name(model_name: str) -> str: - """Format a comparison result name as a user-facing expectation name.""" - - if model_name.startswith("assertion "): - return model_name - return f"expected {model_name}" diff --git a/src/sqlbuild/cli/progress/main/_write_execution_header.py b/src/sqlbuild/cli/progress/main/_write_execution_header.py deleted file mode 100644 index e332773963..0000000000 --- a/src/sqlbuild/cli/progress/main/_write_execution_header.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Generic CLI execution header output.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.cli.progress.main._execution_header import format_execution_header -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.surface_header import format_surface_header - - -def write_execution_header( - *, stream: TextIO, command: str, target: str | None, concurrency: int, use_color: bool -) -> None: - """Write an execution header for command progress output.""" - - style: CliStyle = CliStyle(use_color=use_color) - header: str = format_execution_header( - command=command, - target=target, - concurrency=concurrency, - ) - stream.write(f"{format_surface_header(style=style, title='Execution', context=header)}\n\n") - stream.flush() diff --git a/src/sqlbuild/cli/progress/main/verbose_inspection_output.py b/src/sqlbuild/cli/progress/main/verbose_inspection_output.py index d3eafe222e..84d16e9534 100644 --- a/src/sqlbuild/cli/progress/main/verbose_inspection_output.py +++ b/src/sqlbuild/cli/progress/main/verbose_inspection_output.py @@ -5,8 +5,8 @@ from collections.abc import Iterator from contextlib import contextmanager +from sqlbuild._native import VerboseInspectionWriter from sqlbuild.adapter.relations.main.inspection_query_sink import inspection_query_sink -from sqlbuild.cli.progress.classes.verbose_inspection_writer import VerboseInspectionWriter @contextmanager diff --git a/src/sqlbuild/cli/progress/models.py b/src/sqlbuild/cli/progress/models.py index 3e6eeee7cd..cbefa6e379 100644 --- a/src/sqlbuild/cli/progress/models.py +++ b/src/sqlbuild/cli/progress/models.py @@ -4,8 +4,7 @@ from dataclasses import dataclass -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter +from sqlbuild._native import ConnectionProgressReporter, PlanningProgressReporter from sqlbuild.compiler.auditing.types import AuditOutcome diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 22ee64a1e5..dd611be592 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -5,6 +5,7 @@ import logging from collections.abc import Callable +from sqlbuild._native import get_config_str from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -58,7 +59,6 @@ from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.diagnostics.main.log_sql import log_sql -from sqlbuild.spec.contracts.main.get_config_str import get_config_str def detect_changes( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py index 7021e2161e..dd3f39880d 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -5,6 +5,7 @@ from collections.abc import Mapping from dataclasses import dataclass, replace +from sqlbuild._native import get_config_str from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType @@ -48,7 +49,6 @@ WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import InputColumns, MaterializationType, WarningSeverity -from sqlbuild.spec.contracts.main.get_config_str import get_config_str from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy _HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py index dc31b25b99..548567bdc4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import get_config_str from sqlbuild.adapter.contract.models import ColumnInfo, NormalizedType, RelationInfo from sqlbuild.adapter.contract.types import TypeFamily from sqlbuild.adapter.type_system.main.normalize_type import normalize_type @@ -21,7 +22,6 @@ SchemaChangeKind, SnapshotSchemaChangePolicy, ) -from sqlbuild.spec.contracts.main.get_config_str import get_config_str from sqlbuild.spec.contracts.models import SnapshotsConfig _DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index c7363377c7..5ddc97cc42 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -6,6 +6,7 @@ from collections import defaultdict from typing import Any +from sqlbuild._native import get_config_str from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, @@ -42,7 +43,6 @@ ) from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities -from sqlbuild.spec.contracts.main.get_config_str import get_config_str _HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py index e7dbb735c3..da617d68bc 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py @@ -4,6 +4,7 @@ from collections.abc import Callable, Sequence +from sqlbuild._native import append_overflow_line, count_header_style, visible_entries from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, MigrationCompatibility, @@ -20,10 +21,7 @@ ) from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.append_overflow_line import append_overflow_line -from sqlbuild.presentation.main.count_header import count_header_style from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.main.visible_entries import visible_entries from sqlbuild.presentation.models import DisplayOptions diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py b/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py index 6f98a7a61c..fc5badd57b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py @@ -6,6 +6,7 @@ from datetime import datetime from sqlbuild._native import ( + get_config_str, hard_coded_relation_remedy, match_project_relation, old_name_reference_message, @@ -54,7 +55,6 @@ from sqlbuild.cursor_algebra.models import Duration from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.main.get_config_str import get_config_str from sqlbuild.spec.contracts.models import MigrationsConfig, SchemaColumn _HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index c22114d0f9..f2cc964f6d 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -6,7 +6,7 @@ from dataclasses import replace from datetime import UTC, datetime -from sqlbuild._native import compute_query_hash +from sqlbuild._native import compute_query_hash, get_config_str from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo from sqlbuild.adapter.contract.types import BuiltinAdapter, MigrationTransfer from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY @@ -68,7 +68,6 @@ WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity -from sqlbuild.spec.contracts.main.get_config_str import get_config_str from sqlbuild.spec.contracts.models import SnapshotsConfig from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, TableType diff --git a/src/sqlbuild/compiler/planner/_helpers/output/inclusive_cursor_end.py b/src/sqlbuild/compiler/planner/_helpers/output/inclusive_cursor_end.py index ae0642005a..c01cc73259 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/inclusive_cursor_end.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/inclusive_cursor_end.py @@ -4,13 +4,13 @@ from datetime import datetime +from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive +from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive +from sqlbuild._native import cursor_render as render +from sqlbuild._native import cursor_try_parse as try_parse from sqlbuild.compiler.planner.constants import WHOLE_DAY_CURSOR_GRAINS from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.main.exclusive_to_inclusive import exclusive_to_inclusive -from sqlbuild.cursor_algebra.main.inclusive_to_exclusive import inclusive_to_exclusive -from sqlbuild.cursor_algebra.main.render import render -from sqlbuild.cursor_algebra.main.try_parse import try_parse from sqlbuild.cursor_algebra.types import CursorScalar diff --git a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py index 81397ec4ed..4f13f52740 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py @@ -4,7 +4,7 @@ from datetime import datetime -from sqlbuild._native import resolve_cursor_input_roles +from sqlbuild._native import get_config_str, resolve_cursor_input_roles from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompileSqlReference, @@ -21,7 +21,6 @@ from sqlbuild.compiler.planner.types import CursorType, IncrementalMode from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -from sqlbuild.spec.contracts.main.get_config_str import get_config_str from sqlbuild.spec.contracts.types import MicrobatchLimitAction _CAP_ACTIONS: frozenset[MicrobatchLimitAction] = frozenset( diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 73fd293a0c..b33e29bad2 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -7,7 +7,18 @@ from decimal import Decimal from typing import Any -from sqlbuild._native import render_source_relation, resolve_cursor_input_roles +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import ( + get_config_bool, + get_config_cursor_bound, + get_config_int, + get_config_str, + get_config_string_tuple, + render_source_relation, + resolve_cursor_input_roles, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY @@ -141,16 +152,8 @@ WarningSeverity, ) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.cursor_algebra.models import TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel -from sqlbuild.spec.contracts.main.get_config_bool import get_config_bool -from sqlbuild.spec.contracts.main.get_config_cursor_bound import get_config_cursor_bound -from sqlbuild.spec.contracts.main.get_config_int import get_config_int -from sqlbuild.spec.contracts.main.get_config_str import get_config_str -from sqlbuild.spec.contracts.main.get_config_string_tuple import get_config_string_tuple from sqlbuild.spec.contracts.models import ( FutureCursorsConfig, LocalConfig, diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/config.py b/src/sqlbuild/compiler/planner/_helpers/resolve/config.py index 09533b960d..26698e7dcc 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/config.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/config.py @@ -2,9 +2,9 @@ from __future__ import annotations +from sqlbuild._native import get_config_str from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner.types import IncrementalStrategy -from sqlbuild.spec.contracts.main.get_config_str import get_config_str def get_config_append_cursor_inclusive(model: CompiledModelView) -> bool: diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py index 3efc3ffa7b..c3a7d1450c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py @@ -7,6 +7,14 @@ from dataclasses import replace from datetime import datetime +from sqlbuild._native import cursor_floor_to_grain as floor_to_grain +from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive +from sqlbuild._native import cursor_max_bound as max_bound +from sqlbuild._native import cursor_min_bound as min_bound +from sqlbuild._native import cursor_observed_partition as observed_partition +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_try_parse as try_parse +from sqlbuild._native import get_config_str from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -26,16 +34,8 @@ ) from sqlbuild.cursor_algebra.constants import GRAIN_ORDER from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.main.floor_to_grain import floor_to_grain -from sqlbuild.cursor_algebra.main.inclusive_to_exclusive import inclusive_to_exclusive -from sqlbuild.cursor_algebra.main.max_bound import max_bound -from sqlbuild.cursor_algebra.main.min_bound import min_bound -from sqlbuild.cursor_algebra.main.observed_partition import observed_partition -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.try_parse import try_parse from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar -from sqlbuild.spec.contracts.main.get_config_str import get_config_str _INTEGER_LITERAL: re.Pattern[str] = re.compile(r"[+-]?\d+") _LISTED_MODEL_LIMIT: int = 3 diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py index 1479a470b4..7bbffc5884 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py @@ -2,10 +2,10 @@ from __future__ import annotations +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token def render_cursor_intrinsic_bounds( diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py b/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py index a959212e1d..12c3b57ab3 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py @@ -5,6 +5,11 @@ from dataclasses import replace from datetime import UTC, datetime +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_floor_to_grain as floor_to_grain +from sqlbuild._native import cursor_next_boundary as next_boundary +from sqlbuild._native import cursor_render as render +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError from sqlbuild.compiler.planner.models import ( CursorBounds, @@ -13,11 +18,6 @@ FutureCursorSafetyEvidence, ) from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.floor_to_grain import floor_to_grain -from sqlbuild.cursor_algebra.main.next_boundary import next_boundary -from sqlbuild.cursor_algebra.main.render import render -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.cursor_algebra.models import DateValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.spec.contracts.constants import ZERO_DAY_CURSOR_DURATION diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py b/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py index 7fc9d32c8b..85d9df6458 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py @@ -5,6 +5,11 @@ from dataclasses import replace from datetime import UTC, datetime +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_floor_to_grain as floor_to_grain +from sqlbuild._native import cursor_min_bound as min_bound +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render from sqlbuild.compiler.planner._helpers.resolve.cursor import apply_typed_cursor_replay_policy from sqlbuild.compiler.planner.exceptions import MaximumAutomaticStartError from sqlbuild.compiler.planner.models import ( @@ -19,11 +24,6 @@ CursorType, IncrementalStrategy, ) -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.floor_to_grain import floor_to_grain -from sqlbuild.cursor_algebra.main.min_bound import min_bound -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.models import DateValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.spec.contracts.constants import ZERO_DAY_CURSOR_DURATION diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py index 21e5b6bfd6..1f7a3c182c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py @@ -6,6 +6,7 @@ from collections.abc import Callable, Iterator import sqlbuild._native as _native +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, @@ -34,7 +35,6 @@ ) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token _REF_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.REF) _SEED_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SEED) diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py index 67612fe91d..96a1043867 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py @@ -5,7 +5,12 @@ from dataclasses import replace from datetime import datetime -from sqlbuild._native import assert_no_unresolved_sql_markers, resolve_cursor_input_roles +from sqlbuild._native import ( + assert_no_unresolved_sql_markers, + get_config_cursor_bound, + get_config_str, + resolve_cursor_input_roles, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics @@ -63,8 +68,6 @@ from sqlbuild.compiler.planner.types import BackfillAction, IncrementalMode, MaterializationType from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.cursor_algebra.types import BoundSentinel -from sqlbuild.spec.contracts.main.get_config_cursor_bound import get_config_cursor_bound -from sqlbuild.spec.contracts.main.get_config_str import get_config_str from sqlbuild.spec.contracts.models import FutureCursorsConfig, SourceEntry, StartCursorsConfig diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py index b67bd0242d..1ab40f1157 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py @@ -4,6 +4,7 @@ import re +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import render_source_relation from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo @@ -21,7 +22,6 @@ quoted_reference_call_pattern, ) from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry _SOURCE_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SOURCE) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py index 61e38822c7..3df6a798ef 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py @@ -4,6 +4,11 @@ from dataclasses import dataclass +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_floor_to_grain as floor_to_grain +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render +from sqlbuild._native import get_config_str from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics @@ -15,12 +20,7 @@ from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.floor_to_grain import floor_to_grain -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.types import CursorScalar -from sqlbuild.spec.contracts.main.get_config_str import get_config_str _DEFAULT_TIMESTAMP_START: str = "1900-01-01 00:00:00" _DEFAULT_TIMESTAMP_END: str = "2999-12-31 00:00:00" diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index 3929136861..41dd53f435 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -9,7 +9,14 @@ from datetime import datetime from typing import Any +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive +from sqlbuild._native import cursor_observed_partition as observed_partition +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render from sqlbuild._native import ( + get_config_cursor_bound, + get_config_str, read_latest_fingerprints, render_source_relation, resolve_cursor_input_roles, @@ -91,16 +98,9 @@ from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.exclusive_to_inclusive import exclusive_to_inclusive -from sqlbuild.cursor_algebra.main.observed_partition import observed_partition -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.models import DateValue from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.spec.contracts.main.get_config_cursor_bound import get_config_cursor_bound -from sqlbuild.spec.contracts.main.get_config_str import get_config_str from sqlbuild.spec.contracts.models import SourceEntry, StartCursorsConfig _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.planner") diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 7aedfb065d..f849e52563 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.cursor_algebra.types import BoundSentinel from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy diff --git a/src/sqlbuild/compiler/planner/exceptions.py b/src/sqlbuild/compiler/planner/exceptions.py index d9fecd255c..9b3763810f 100644 --- a/src/sqlbuild/compiler/planner/exceptions.py +++ b/src/sqlbuild/compiler/planner/exceptions.py @@ -4,7 +4,7 @@ from typing import Any -from sqlbuild.cursor_algebra.main.render import render +from sqlbuild._native import cursor_render as render class PlannerInputError(ValueError): diff --git a/src/sqlbuild/compiler/planner/main/execution/display_plan.py b/src/sqlbuild/compiler/planner/main/execution/display_plan.py index bb59e1a16b..b962f8575d 100644 --- a/src/sqlbuild/compiler/planner/main/execution/display_plan.py +++ b/src/sqlbuild/compiler/planner/main/execution/display_plan.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import get_config_str from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) @@ -19,7 +20,6 @@ PlanAction, PlanReason, ) -from sqlbuild.spec.contracts.main.get_config_str import get_config_str def build_display_only_sqlbuild_plan( diff --git a/src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py b/src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py index 080317d56d..6bd19aadcb 100644 --- a/src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py +++ b/src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import cursor_render as render from sqlbuild.compiler.planner.models import CursorBounds, FutureCursorSafetyEvidence -from sqlbuild.cursor_algebra.main.render import render def future_cursor_cap_warning(bounds: CursorBounds | None) -> str | None: diff --git a/src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py b/src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py index a93b60c68a..061ec067fd 100644 --- a/src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py +++ b/src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import cursor_render as render from sqlbuild.compiler.planner.models import CursorBounds, MaximumStartSafetyEvidence -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.types import CursorScalar diff --git a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py index 8797e0d342..5caddd6e1f 100644 --- a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py +++ b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py @@ -2,10 +2,9 @@ from __future__ import annotations +from sqlbuild._native import append_overflow_line, visible_entries from sqlbuild.compiler.planner._helpers.migrations.display import format_model_migrations from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.presentation.main.append_overflow_line import append_overflow_line -from sqlbuild.presentation.main.visible_entries import visible_entries from sqlbuild.presentation.models import DisplayOptions diff --git a/src/sqlbuild/compiler/planner/main/selection/selector_match.py b/src/sqlbuild/compiler/planner/main/selection/_selector_match.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/selection/selector_match.py rename to src/sqlbuild/compiler/planner/main/selection/_selector_match.py diff --git a/src/sqlbuild/compiler/planner/main/selection/selector_tokens.py b/src/sqlbuild/compiler/planner/main/selection/_selector_tokens.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/selection/selector_tokens.py rename to src/sqlbuild/compiler/planner/main/selection/_selector_tokens.py diff --git a/src/sqlbuild/compiler/planner/main/selection/upstream.py b/src/sqlbuild/compiler/planner/main/selection/_upstream.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/selection/upstream.py rename to src/sqlbuild/compiler/planner/main/selection/_upstream.py diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index df139f94a6..135520c10a 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any +from sqlbuild._native import cursor_parse as parse from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RetentionRequest from sqlbuild.adapter.contract.types import MigrationTransfer @@ -72,7 +73,6 @@ DirectSourceFreshnessPlanningResult, ) from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.main.parse import parse from sqlbuild.cursor_algebra.models import ( DateValue, IntegerValue, diff --git a/src/sqlbuild/compiler/refactoring/constants.py b/src/sqlbuild/compiler/refactoring/constants.py deleted file mode 100644 index 7be939d5b9..0000000000 --- a/src/sqlbuild/compiler/refactoring/constants.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Constants for project refactorings.""" - -from __future__ import annotations - -from sqlbuild.compiler.refactoring.types import ( - RefactorOperation, -) - -MODEL_KIND_PREFIX: str = "model:" -MIGRATE_FROM_KEY: str = "migrate_from" -OPERATION_TITLES: dict[RefactorOperation, str] = { - RefactorOperation.RENAME_MODEL: "Rename model", - RefactorOperation.MOVE_MODEL: "Move model", - RefactorOperation.RENAME_COLUMN: "Rename column", -} -GENERIC_DIALECT: str = "generic" diff --git a/src/sqlbuild/compiler/refactoring/exceptions.py b/src/sqlbuild/compiler/refactoring/exceptions.py index 4769fbb810..87a4e53647 100644 --- a/src/sqlbuild/compiler/refactoring/exceptions.py +++ b/src/sqlbuild/compiler/refactoring/exceptions.py @@ -3,30 +3,6 @@ from __future__ import annotations -class RefactorInputError(ValueError): - """A rename or move request that cannot be planned at all.""" - - code: str = "C950" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help - - -class RefactorEditError(RefactorInputError): - """Two planned edits overlap, which a correct plan never produces.""" - - code: str = "C957" - - -class RefactorWriteError(RefactorInputError): - """Project files changed between planning and writing.""" - - code: str = "C958" - - class RefactorValueError(ValueError): """An unhandled value failure the Python planner also leaves to the caller.""" diff --git a/src/sqlbuild/compiler/refactoring/models.py b/src/sqlbuild/compiler/refactoring/models.py index b732582931..577f489a22 100644 --- a/src/sqlbuild/compiler/refactoring/models.py +++ b/src/sqlbuild/compiler/refactoring/models.py @@ -2,15 +2,11 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.refactoring.types import ( - EditKind, - RefactorOperation, -) @dataclass(frozen=True) @@ -22,45 +18,6 @@ class RefactorProject: discovered: DiscoveredProjectInputs -@dataclass(frozen=True) -class RefactorRequest: - """One requested rename or move.""" - - operation: RefactorOperation - model_name: str - new_name: str - column_name: str | None = None - destination: str | None = None - cascade: bool = False - - -@dataclass(frozen=True) -class TextEdit: - """One replacement in an authored file, in offsets of the file before any edit.""" - - start: int - end: int - replacement: str - kind: EditKind - line: int - column: int - before: str - after: str - - -@dataclass(frozen=True) -class FileChange: - """Every edit to one file, and where the file lives afterwards.""" - - path: str - original_path: str - edits: tuple[TextEdit, ...] = field(default_factory=tuple) - - @property - def moved(self) -> bool: - return self.path != self.original_path - - @dataclass(frozen=True) class ManualLocation: """A location a refactoring cannot rewrite safely.""" @@ -69,25 +26,3 @@ class ManualLocation: line: int | None column: int | None reason: str - - -@dataclass(frozen=True) -class MigrationDeclaration: - """A migrate_from declaration the refactoring adds to keep history.""" - - model_name: str - declaration: str - reason: str - - -@dataclass(frozen=True) -class RefactorPlan: - """Every edit, manual location, and migration of one refactoring.""" - - request: RefactorRequest - changes: tuple[FileChange, ...] = field(default_factory=tuple) - manual: tuple[ManualLocation, ...] = field(default_factory=tuple) - blocking: tuple[ManualLocation, ...] = field(default_factory=tuple) - migrations: tuple[MigrationDeclaration, ...] = field(default_factory=tuple) - renamed_columns: tuple[tuple[str, str, str], ...] = field(default_factory=tuple) - help: str | None = None diff --git a/src/sqlbuild/compiler/refactoring/types.py b/src/sqlbuild/compiler/refactoring/types.py deleted file mode 100644 index f92d584113..0000000000 --- a/src/sqlbuild/compiler/refactoring/types.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Enums and type aliases for project refactorings.""" - -from __future__ import annotations - -from enum import StrEnum - - -class RefactorOperation(StrEnum): - """One supported refactoring.""" - - RENAME_MODEL = "rename_model" - MOVE_MODEL = "move_model" - RENAME_COLUMN = "rename_column" - - -class RefactorStatus(StrEnum): - """Terminal state of one refactoring command.""" - - APPLIED = "applied" - DRY_RUN = "dry_run" - REFUSED = "refused" - COMPILE_FAILED = "compile_failed" - - -class NativeRefactorErrorKind(StrEnum): - """Which exception a native refactoring failure becomes.""" - - INPUT = "input" - EDIT = "edit" - WRITE = "write" - VALUE = "value" - IO = "io" - - -class EditKind(StrEnum): - """What one text edit changes, for output grouping.""" - - REFERENCE = "reference" - FIXTURE = "fixture" - HEADER = "header" - COLUMN = "column" - MIGRATION = "migration" - - -class SqlFileRole(StrEnum): - """What an authored SQL file declares.""" - - MODEL = "model" - TEST = "test" - SCENARIO = "scenario" - AUDIT = "audit" - HOOK = "hook" - FUNCTION = "function" - SCHEMA = "schema" - YAML = "yaml" diff --git a/src/sqlbuild/cost/_helpers/store.py b/src/sqlbuild/cost/_helpers/store.py deleted file mode 100644 index 7a9d929a35..0000000000 --- a/src/sqlbuild/cost/_helpers/store.py +++ /dev/null @@ -1,283 +0,0 @@ -"""Atomic project-local run-cost persistence.""" - -from __future__ import annotations - -import json -import os -from dataclasses import asdict -from datetime import datetime -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any, cast - -from sqlbuild.cost.constants import LATEST_COST_RUN_SELECTOR, RUNNING_BUILD_STATUS -from sqlbuild.cost.exceptions import CostArtifactError -from sqlbuild.cost.models import CostRunRecord, ResourceCost, RunCostSummary -from sqlbuild.cost.types import CostStatus - -_COST_ARTIFACT_VERSION: int = 1 -_UNSAFE_RUN_ID_COMPONENTS: frozenset[str] = frozenset({".", ".."}) -_RUN_ID_PATH_SEPARATORS: tuple[str, ...] = ("/", "\\") - - -class _Encoder(json.JSONEncoder): - def default(self, o: Any) -> Any: - if isinstance(o, Decimal): - return str(o) - if isinstance(o, datetime): - return o.isoformat() - if isinstance(o, CostStatus): - return o.value - return super().default(o) - - -def write_cost_run(*, project_dir: Path, record: CostRunRecord) -> Path: - run_dir: Path = _run_directory(project_dir=project_dir, run_id=record.run_id) - run_dir.mkdir(parents=True, exist_ok=True) - destination: Path = run_dir / "run.json" - temporary: Path = run_dir / ".run.json.tmp" - temporary.write_text( - json.dumps(asdict(record), cls=_Encoder, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - os.replace(temporary, destination) - return destination - - -def format_cost_json(record: CostRunRecord) -> str: - return json.dumps(cost_output_payload(record=record), cls=_Encoder, indent=2, sort_keys=True) - - -def format_cost_history_json( - *, records: tuple[CostRunRecord, ...], matching_count: int | None = None -) -> str: - return json.dumps( - { - "schema_version": 1, - "command": "cost_history", - "count": len(records), - "matching_count": len(records) if matching_count is None else matching_count, - "runs": [cost_output_payload(record=record) for record in records], - }, - cls=_Encoder, - indent=2, - sort_keys=True, - ) - - -def read_cost_run(*, project_dir: Path, run_id: str) -> CostRunRecord | None: - path: Path = _run_directory(project_dir=project_dir, run_id=run_id) / "run.json" - if not path.is_file(): - return None - record: CostRunRecord = _record_from_payload(json.loads(path.read_text(encoding="utf-8"))) - if record.run_id != run_id: - raise CostArtifactError("cost run artifact run ID does not match its directory") - return record - - -def list_cost_runs(*, project_dir: Path) -> tuple[CostRunRecord, ...]: - root: Path = project_dir / "target" / "executions" - if not root.is_dir(): - return () - records: list[CostRunRecord] = [] - for path in root.glob("*/run.json"): - try: - if not path.resolve().is_relative_to(root.resolve()): - continue - record: CostRunRecord = _record_from_payload( - json.loads(path.read_text(encoding="utf-8")) - ) - if record.run_id != path.parent.name: - raise CostArtifactError("cost run artifact run ID does not match its directory") - records.append(record) - except (OSError, ValueError, TypeError, KeyError, InvalidOperation, json.JSONDecodeError): - continue - records.sort(key=lambda record: (record.completed_at, record.run_id), reverse=True) - return tuple(records) - - -def resolve_cost_run(*, project_dir: Path, selector: str) -> CostRunRecord | None: - records: tuple[CostRunRecord, ...] = list_cost_runs(project_dir=project_dir) - if selector == LATEST_COST_RUN_SELECTOR: - return next( - (record for record in records if record.build_status != RUNNING_BUILD_STATUS), - None, - ) - matches: tuple[CostRunRecord, ...] = tuple( - record for record in records if record.run_id.startswith(selector) - ) - return matches[0] if len(matches) == 1 else None - - -def _record_from_payload(payload: object) -> CostRunRecord: - if not isinstance(payload, dict): - raise CostArtifactError("cost run artifact must be an object") - payload_dict: dict[str, Any] = cast(dict[str, Any], payload) - version: int = int(payload_dict["version"]) - if version != _COST_ARTIFACT_VERSION: - raise CostArtifactError(f"unsupported cost run artifact version: {version}") - run_id: str = str(payload_dict["run_id"]) - _validate_run_id(run_id=run_id) - cost_payload: object = payload_dict["cost"] - if not isinstance(cost_payload, dict): - raise CostArtifactError("cost run artifact 'cost' must be an object") - cost_payload_dict: dict[str, Any] = cast(dict[str, Any], cost_payload) - resources_payload: object = cost_payload_dict.get("resources", []) - if not isinstance(resources_payload, list): - raise CostArtifactError("cost run artifact 'resources' must be a list") - if not all(isinstance(item, dict) for item in resources_payload): - raise CostArtifactError("cost run artifact resources must be objects") - resources: tuple[ResourceCost, ...] = tuple( - ResourceCost( - resource_type=str(cast(dict[str, Any], item)["resource_type"]), - resource_name=str(cast(dict[str, Any], item)["resource_name"]), - warehouse_name=str(cast(dict[str, Any], item)["warehouse_name"]), - warehouse_size=str(cast(dict[str, Any], item)["warehouse_size"]), - warehouse_type=( - None - if cast(dict[str, Any], item).get("warehouse_type") is None - else str(cast(dict[str, Any], item)["warehouse_type"]) - ), - cluster_number=( - None - if cast(dict[str, Any], item).get("cluster_number") is None - else int(cast(dict[str, Any], item)["cluster_number"]) - ), - query_count=int(cast(dict[str, Any], item)["query_count"]), - attributed_seconds=Decimal(str(cast(dict[str, Any], item)["attributed_seconds"])), - bytes_scanned=int(cast(dict[str, Any], item)["bytes_scanned"]), - estimated_compute_credits=Decimal( - str(cast(dict[str, Any], item)["estimated_compute_credits"]) - ), - estimated_usd=Decimal(str(cast(dict[str, Any], item)["estimated_usd"])), - ) - for item in resources_payload - if isinstance(item, dict) - ) - started_at: datetime = _aware_datetime(value=payload_dict["started_at"], field="started_at") - completed_at: datetime = _aware_datetime( - value=payload_dict["completed_at"], field="completed_at" - ) - had_executable_work_payload: object = payload_dict.get("had_executable_work") - if had_executable_work_payload is not None and not isinstance( - had_executable_work_payload, bool - ): - raise CostArtifactError("cost run artifact 'had_executable_work' must be a boolean") - cost: RunCostSummary = RunCostSummary( - status=CostStatus(str(cost_payload_dict["status"])), - usd_per_credit=Decimal(str(cost_payload_dict["usd_per_credit"])), - rate_source=str(cost_payload_dict["rate_source"]), - resources=resources, - query_count=int(cost_payload_dict.get("query_count", 0)), - attributed_seconds=Decimal(str(cost_payload_dict.get("attributed_seconds", 0))), - bytes_scanned=int(cost_payload_dict.get("bytes_scanned", 0)), - estimated_compute_credits=Decimal( - str(cost_payload_dict.get("estimated_compute_credits", 0)) - ), - estimated_usd=Decimal(str(cost_payload_dict.get("estimated_usd", 0))), - expected_statement_count=int(cost_payload_dict.get("expected_statement_count", 0)), - observed_statement_count=int(cost_payload_dict.get("observed_statement_count", 0)), - missing_statement_count=int(cost_payload_dict.get("missing_statement_count", 0)), - skipped_statement_count=int(cost_payload_dict.get("skipped_statement_count", 0)), - source=str(cost_payload_dict.get("source", "")), - method=str(cost_payload_dict.get("method", "")), - limitations=tuple(str(value) for value in cost_payload_dict.get("limitations", [])), - message=( - None if cost_payload_dict.get("message") is None else str(cost_payload_dict["message"]) - ), - ) - return CostRunRecord( - version=version, - run_id=run_id, - adapter_name=str(payload_dict["adapter_name"]), - target_name=( - None if payload_dict.get("target_name") is None else str(payload_dict["target_name"]) - ), - build_status=str(payload_dict["build_status"]), - started_at=started_at, - completed_at=completed_at, - cost=cost, - had_executable_work=( - None if had_executable_work_payload is None else had_executable_work_payload - ), - ) - - -def _run_directory(*, project_dir: Path, run_id: str) -> Path: - _validate_run_id(run_id=run_id) - root: Path = project_dir / "target" / "executions" - run_dir: Path = root / run_id - if not run_dir.resolve().is_relative_to(root.resolve()): - raise CostArtifactError("cost run path must remain under the project execution directory") - return run_dir - - -def _validate_run_id(*, run_id: str) -> None: - if ( - not run_id - or run_id in _UNSAFE_RUN_ID_COMPONENTS - or Path(run_id).name != run_id - or any(separator in run_id for separator in _RUN_ID_PATH_SEPARATORS) - ): - raise CostArtifactError("cost run ID must be one path component") - - -def _aware_datetime(*, value: object, field: str) -> datetime: - parsed: datetime = datetime.fromisoformat(str(value)) - if parsed.tzinfo is None or parsed.utcoffset() is None: - raise CostArtifactError(f"cost run artifact '{field}' must include a timezone") - return parsed - - -def cost_output_payload(*, record: CostRunRecord) -> dict[str, Any]: - per_model: list[dict[str, object]] = [ - { - "resource_type": resource.resource_type, - "model": resource.resource_name, - "warehouse": resource.warehouse_name, - "warehouse_size": resource.warehouse_size, - "warehouse_type": resource.warehouse_type, - "cluster_number": resource.cluster_number, - "query_count": resource.query_count, - "attributed_seconds": str(resource.attributed_seconds), - "bytes_scanned": resource.bytes_scanned, - "attributed_compute_credits": str(resource.estimated_compute_credits), - "estimated_usd": str(resource.estimated_usd), - } - for resource in record.cost.resources - ] - return { - "schema_version": 1, - "command": "cost", - "run_id": record.run_id, - "build_status": record.build_status, - "started_at": record.started_at.isoformat(), - "completed_at": record.completed_at.isoformat(), - "adapter": record.adapter_name, - "target": record.target_name, - "had_executable_work": record.had_executable_work, - "status": record.cost.status.value, - "source": record.cost.source, - "currency": "USD", - "usd_per_credit": str(record.cost.usd_per_credit), - "rate_source": record.cost.rate_source, - "method": record.cost.method, - "limitations": list(record.cost.limitations), - "message": record.cost.message, - "coverage": { - "query_count": record.cost.query_count, - "model_rows": len(record.cost.resources), - "expected_statement_count": record.cost.expected_statement_count, - "observed_statement_count": record.cost.observed_statement_count, - "missing_statement_count": record.cost.missing_statement_count, - "skipped_statement_count": record.cost.skipped_statement_count, - }, - "per_model": per_model, - "totals": { - "query_count": record.cost.query_count, - "attributed_seconds": str(record.cost.attributed_seconds), - "bytes_scanned": record.cost.bytes_scanned, - "attributed_compute_credits": str(record.cost.estimated_compute_credits), - "estimated_usd": str(record.cost.estimated_usd), - }, - } diff --git a/src/sqlbuild/cost/classes/run_cost_store.py b/src/sqlbuild/cost/classes/run_cost_store.py deleted file mode 100644 index f7361bc25e..0000000000 --- a/src/sqlbuild/cost/classes/run_cost_store.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Project-local run-cost store.""" - -from pathlib import Path - -from sqlbuild.cost._helpers.store import ( - cost_output_payload, - format_cost_history_json, - format_cost_json, - list_cost_runs, - read_cost_run, - resolve_cost_run, - write_cost_run, -) -from sqlbuild.cost.models import CostRunRecord - - -class RunCostStore: - @staticmethod - def write(*, project_dir: Path, record: CostRunRecord) -> Path: - return write_cost_run(project_dir=project_dir, record=record) - - @staticmethod - def read(*, project_dir: Path, run_id: str) -> CostRunRecord | None: - return read_cost_run(project_dir=project_dir, run_id=run_id) - - @staticmethod - def list(*, project_dir: Path) -> tuple[CostRunRecord, ...]: - return list_cost_runs(project_dir=project_dir) - - @staticmethod - def resolve(*, project_dir: Path, selector: str) -> CostRunRecord | None: - return resolve_cost_run(project_dir=project_dir, selector=selector) - - @staticmethod - def format_json(record: CostRunRecord) -> str: - return format_cost_json(record) - - @staticmethod - def format_history_json( - *, records: tuple[CostRunRecord, ...], matching_count: int | None = None - ) -> str: - return format_cost_history_json(records=records, matching_count=matching_count) - - @staticmethod - def output_payload(*, record: CostRunRecord) -> dict[str, object]: - return cost_output_payload(record=record) diff --git a/src/sqlbuild/cursor_algebra/_helpers/__init__.py b/src/sqlbuild/cursor_algebra/_helpers/__init__.py deleted file mode 100644 index 05fb89b7b6..0000000000 --- a/src/sqlbuild/cursor_algebra/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Private cursor algebra helpers.""" diff --git a/src/sqlbuild/cursor_algebra/_helpers/arithmetic.py b/src/sqlbuild/cursor_algebra/_helpers/arithmetic.py deleted file mode 100644 index 623d852c6a..0000000000 --- a/src/sqlbuild/cursor_algebra/_helpers/arithmetic.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Private scalar arithmetic primitives.""" - -from datetime import date, datetime, timedelta - -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra.constants import GRAIN_FIXED_STEP -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -from sqlbuild.cursor_algebra.types import CursorScalar - -_FINAL_MONTH: int = 12 - - -def floor_scalar(*, value: CursorScalar, grain: CursorGrain) -> CursorScalar: - """Floor one scalar without crossing the serialization boundary.""" - - if isinstance(value, IntegerValue): - return value - if isinstance(value, DateValue): - plain_date: date = value.value - if grain == CursorGrain.MONTH: - return DateValue(value=plain_date.replace(day=1)) - if grain == CursorGrain.YEAR: - return DateValue(value=plain_date.replace(month=1, day=1)) - return value - timestamp: datetime = value.value - if grain == CursorGrain.SECOND: - timestamp = timestamp.replace(microsecond=0) - elif grain == CursorGrain.MINUTE: - timestamp = timestamp.replace(second=0, microsecond=0) - elif grain == CursorGrain.HOUR: - timestamp = timestamp.replace(minute=0, second=0, microsecond=0) - elif grain == CursorGrain.DAY: - timestamp = timestamp.replace(hour=0, minute=0, second=0, microsecond=0) - elif grain == CursorGrain.MONTH: - timestamp = timestamp.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - elif grain == CursorGrain.YEAR: - timestamp = timestamp.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0) - return TimestampValue(value=timestamp) - - -def advance_scalar(*, value: CursorScalar, grain: CursorGrain) -> CursorScalar: - """Advance one aligned scalar by one grain.""" - - if isinstance(value, IntegerValue): - return IntegerValue(value=value.value + 1) - if isinstance(value, DateValue) and grain not in { - CursorGrain.MONTH, - CursorGrain.YEAR, - }: - return DateValue(value=value.value + timedelta(days=1)) - fixed_step: timedelta | None = GRAIN_FIXED_STEP[grain] - raw: date | datetime = value.value - if fixed_step is not None: - advanced: date | datetime = raw + fixed_step - elif grain == CursorGrain.MONTH: - advanced = raw.replace( - year=raw.year + (1 if raw.month == _FINAL_MONTH else 0), - month=1 if raw.month == _FINAL_MONTH else raw.month + 1, - day=1, - ) - else: - advanced = raw.replace(year=raw.year + 1, month=1, day=1) - if isinstance(value, TimestampValue) and isinstance(advanced, datetime): - return TimestampValue(value=advanced) - if isinstance(advanced, datetime): - return DateValue(value=advanced.date()) - return DateValue(value=advanced) - - -def advance_scalar_by( - *, value: CursorScalar, grain: CursorGrain | None, steps: int -) -> CursorScalar: - """Advance an aligned scalar by many grain units in constant time.""" - - if isinstance(value, IntegerValue): - return IntegerValue(value=value.value + steps) - if grain is None: - raise CursorAlgebraError("temporal cursor values require a grain") - if isinstance(value, DateValue): - if grain == CursorGrain.MONTH: - return DateValue(value=_advance_months(value=value.value, steps=steps)) - if grain == CursorGrain.YEAR: - return DateValue(value=value.value.replace(year=value.value.year + steps)) - return DateValue(value=value.value + timedelta(days=steps)) - if grain == CursorGrain.MONTH: - advanced_month: date | datetime = _advance_months(value=value.value, steps=steps) - if not isinstance(advanced_month, datetime): - raise CursorAlgebraError("timestamp month advancement lost timestamp representation") - return TimestampValue(value=advanced_month) - if grain == CursorGrain.YEAR: - return TimestampValue(value=value.value.replace(year=value.value.year + steps)) - fixed_step: timedelta | None = GRAIN_FIXED_STEP[grain] - if fixed_step is None: - raise CursorAlgebraError(f"cursor grain has no fixed step: {grain}") - return TimestampValue(value=value.value + fixed_step * steps) - - -def _advance_months(*, value: date | datetime, steps: int) -> date | datetime: - month_index: int = value.year * _FINAL_MONTH + value.month - 1 + steps - year, zero_based_month = divmod(month_index, _FINAL_MONTH) - return value.replace(year=year, month=zero_based_month + 1, day=1) diff --git a/src/sqlbuild/cursor_algebra/_helpers/comparison.py b/src/sqlbuild/cursor_algebra/_helpers/comparison.py deleted file mode 100644 index a291bc445b..0000000000 --- a/src/sqlbuild/cursor_algebra/_helpers/comparison.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Private typed cursor comparison helpers.""" - -from datetime import UTC, datetime - -from sqlbuild.cursor_algebra._helpers.parsing import parse_scalar -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -from sqlbuild.cursor_algebra.types import CursorScalar - - -def comparison_value(*, value: CursorScalar) -> datetime | int: - """Project a cursor scalar into a sortable domain.""" - - if isinstance(value, IntegerValue): - return value.value - if isinstance(value, DateValue): - return datetime.combine(value.value, datetime.min.time()) - timestamp: datetime = value.value - if timestamp.tzinfo is not None: - return timestamp.astimezone(UTC).replace(tzinfo=None) - return timestamp - - -def compare_scalars(*, left: CursorScalar, right: CursorScalar) -> int: - """Compare two compatible cursor scalars.""" - - left_value: datetime | int = comparison_value(value=left) - right_value: datetime | int = comparison_value(value=right) - if isinstance(left_value, int) and isinstance(right_value, int): - return (left_value > right_value) - (left_value < right_value) - if isinstance(left_value, datetime) and isinstance(right_value, datetime): - return (left_value > right_value) - (left_value < right_value) - raise CursorAlgebraError("cannot compare temporal and integer cursor values") - - -def as_scalar(*, raw: object, cursor_type: str) -> CursorScalar: - """Return a typed value unchanged or parse its boundary representation.""" - - if isinstance(raw, TimestampValue | DateValue | IntegerValue): - return raw - return parse_scalar(raw=raw, cursor_type=cursor_type) diff --git a/src/sqlbuild/cursor_algebra/_helpers/intervals.py b/src/sqlbuild/cursor_algebra/_helpers/intervals.py deleted file mode 100644 index d4e86bc936..0000000000 --- a/src/sqlbuild/cursor_algebra/_helpers/intervals.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Private half-open cursor interval algebra.""" - -from datetime import datetime, timedelta - -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra._helpers.arithmetic import ( - advance_scalar, - advance_scalar_by, - floor_scalar, -) -from sqlbuild.cursor_algebra._helpers.comparison import compare_scalars, comparison_value -from sqlbuild.cursor_algebra.constants import GRAIN_FIXED_STEP -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import ( - AlignedInterval, - DateValue, - IntegerValue, - TimestampValue, -) -from sqlbuild.cursor_algebra.types import CursorScalar - - -def observed_partition_value(*, value: CursorScalar, grain: CursorGrain | None) -> AlignedInterval: - """Return the canonical partition containing a physical value.""" - - if isinstance(value, IntegerValue): - return AlignedInterval(start=value, end=IntegerValue(value=value.value + 1), grain=None) - if grain is None: - raise CursorAlgebraError("temporal observed partitions require a grain") - if isinstance(value, DateValue) and grain in { - CursorGrain.SECOND, - CursorGrain.MINUTE, - CursorGrain.HOUR, - }: - value = TimestampValue(value=datetime.combine(value.value, datetime.min.time())) - start: CursorScalar = floor_scalar(value=value, grain=grain) - return AlignedInterval(start=start, end=advance_scalar(value=start, grain=grain), grain=grain) - - -def shift_inclusive_bound( - *, value: CursorScalar, grain: CursorGrain | None, direction: int -) -> CursorScalar: - """Shift an inclusive/exclusive boundary by one compatibility unit.""" - - if isinstance(value, IntegerValue): - return IntegerValue(value=value.value + direction) - if grain is None: - raise CursorAlgebraError("temporal inclusive bounds require a grain") - if isinstance(value, DateValue): - return DateValue(value=value.value + timedelta(days=direction)) - step: timedelta = GRAIN_FIXED_STEP[grain] or timedelta(days=1) - return TimestampValue(value=value.value + direction * step) - - -def clamp_interval(*, interval: AlignedInterval, bounds: AlignedInterval) -> AlignedInterval | None: - """Intersect two canonical intervals.""" - - start: CursorScalar = ( - interval.start - if compare_scalars(left=interval.start, right=bounds.start) >= 0 - else bounds.start - ) - end: CursorScalar = ( - interval.end if compare_scalars(left=interval.end, right=bounds.end) <= 0 else bounds.end - ) - if compare_scalars(left=start, right=end) >= 0: - return None - return AlignedInterval(start=start, end=end, grain=interval.grain) - - -def merge_interval_values(*, intervals: tuple[AlignedInterval, ...]) -> tuple[AlignedInterval, ...]: - """Merge overlapping or adjacent canonical intervals.""" - - ordered: list[AlignedInterval] = sorted( - intervals, key=lambda interval: comparison_value(value=interval.start) - ) - merged: list[AlignedInterval] = [] - for interval in ordered: - if not merged or compare_scalars(left=merged[-1].end, right=interval.start) < 0: - merged.append(interval) - elif compare_scalars(left=merged[-1].end, right=interval.end) < 0: - merged[-1] = AlignedInterval( - start=merged[-1].start, end=interval.end, grain=interval.grain - ) - return tuple(merged) - - -def split_interval(*, interval: AlignedInterval, step: int) -> tuple[AlignedInterval, ...]: - """Split a canonical interval by a positive number of grain units.""" - - if step <= 0: - raise CursorAlgebraError("cursor batch step must be positive") - batches: list[AlignedInterval] = [] - start: CursorScalar = interval.start - while compare_scalars(left=start, right=interval.end) < 0: - end: CursorScalar = advance_scalar_by(value=start, grain=interval.grain, steps=step) - if compare_scalars(left=end, right=interval.end) > 0: - end = interval.end - batches.append(AlignedInterval(start=start, end=end, grain=interval.grain)) - start = end - return tuple(batches) diff --git a/src/sqlbuild/cursor_algebra/_helpers/parsing.py b/src/sqlbuild/cursor_algebra/_helpers/parsing.py deleted file mode 100644 index 2dde3702f8..0000000000 --- a/src/sqlbuild/cursor_algebra/_helpers/parsing.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Private cursor scalar parsing and rendering.""" - -import re -from datetime import UTC, date, datetime -from decimal import Decimal, InvalidOperation - -from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -from sqlbuild.cursor_algebra.types import CursorScalar - -_TRAILING_TIMEZONE_SEPARATOR: re.Pattern[str] = re.compile(r"\s+(?=(?:Z|[+-]\d{2}:?\d{2})$)") - - -def parse_scalar(*, raw: object, cursor_type: str) -> CursorScalar: - """Parse one scalar while preserving temporal representation.""" - - if cursor_type == CursorType.INTEGER: - try: - decimal_value: Decimal = Decimal(str(raw)) - integer_value: int = int(decimal_value) - except (InvalidOperation, ValueError, OverflowError) as error: - raise CursorAlgebraError(f"invalid integer cursor value: {raw}") from error - if decimal_value != integer_value: - raise CursorAlgebraError(f"non-integral integer cursor value: {raw}") - return IntegerValue(value=integer_value) - if cursor_type != CursorType.TIMESTAMP: - raise CursorAlgebraError(f"unsupported cursor type: {cursor_type}") - if isinstance(raw, datetime): - return TimestampValue(value=_normalize_timestamp(value=raw)) - if isinstance(raw, date): - return DateValue(value=raw) - if not isinstance(raw, str): - raise CursorAlgebraError(f"invalid timestamp cursor value: {raw}") - normalized_text: str = _normalize_timestamp_text(value=raw) - try: - return DateValue(value=date.fromisoformat(normalized_text)) - except ValueError: - try: - return TimestampValue( - value=_normalize_timestamp(value=datetime.fromisoformat(normalized_text)), - source_text=normalized_text, - ) - except ValueError as error: - raise CursorAlgebraError(f"invalid timestamp cursor value: {raw}") from error - - -def render_scalar(*, value: CursorScalar) -> str: - """Render one typed cursor scalar.""" - - if isinstance(value, IntegerValue): - return str(value.value) - if isinstance(value, TimestampValue) and value.source_text is not None: - return value.source_text - return value.value.isoformat() - - -def _normalize_timestamp(*, value: datetime) -> datetime: - """Normalize aware timestamps to UTC while leaving presumed-UTC naive values unchanged.""" - - return value.astimezone(UTC) if value.tzinfo is not None else value - - -def _normalize_timestamp_text(*, value: str) -> str: - """Normalize warehouse timestamp text into an ISO-compatible spelling.""" - - return _TRAILING_TIMEZONE_SEPARATOR.sub("", value.strip()) diff --git a/src/sqlbuild/cursor_algebra/constants.py b/src/sqlbuild/cursor_algebra/constants.py index cd243f2ef6..f3dac3ee9a 100644 --- a/src/sqlbuild/cursor_algebra/constants.py +++ b/src/sqlbuild/cursor_algebra/constants.py @@ -1,7 +1,5 @@ """Canonical cursor grain metadata.""" -from datetime import timedelta - from sqlbuild.compiler.planner.types import CursorGrain GRAIN_ORDER: dict[CursorGrain, int] = { @@ -22,15 +20,6 @@ CursorGrain.YEAR: "1y", } -GRAIN_FIXED_STEP: dict[CursorGrain, timedelta | None] = { - CursorGrain.SECOND: timedelta(seconds=1), - CursorGrain.MINUTE: timedelta(minutes=1), - CursorGrain.HOUR: timedelta(hours=1), - CursorGrain.DAY: timedelta(days=1), - CursorGrain.MONTH: None, - CursorGrain.YEAR: None, -} - DURATION_YEAR_UNIT: str = "y" DURATION_MONTH_UNIT: str = "mo" DURATION_DAY_UNIT: str = "d" diff --git a/src/sqlbuild/cursor_algebra/main/cap_from_end.py b/src/sqlbuild/cursor_algebra/main/cap_from_end.py deleted file mode 100644 index 349422f34b..0000000000 --- a/src/sqlbuild/cursor_algebra/main/cap_from_end.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Cap typed cursor intervals from the end.""" - - -def cap_from_end[T](*, intervals: tuple[T, ...], count: int) -> tuple[T, ...]: - """Select at most count intervals from the end.""" - - return intervals[-max(count, 0) :] if count > 0 else () diff --git a/src/sqlbuild/cursor_algebra/main/cap_from_start.py b/src/sqlbuild/cursor_algebra/main/cap_from_start.py deleted file mode 100644 index 035739d883..0000000000 --- a/src/sqlbuild/cursor_algebra/main/cap_from_start.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Cap typed cursor intervals from the start.""" - - -def cap_from_start[T](*, intervals: tuple[T, ...], count: int) -> tuple[T, ...]: - """Select at most count intervals from the start.""" - - return intervals[: max(count, 0)] diff --git a/src/sqlbuild/cursor_algebra/main/clamp.py b/src/sqlbuild/cursor_algebra/main/clamp.py deleted file mode 100644 index 26e25c6820..0000000000 --- a/src/sqlbuild/cursor_algebra/main/clamp.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Clamp a typed cursor interval.""" - -from sqlbuild.cursor_algebra._helpers.intervals import clamp_interval -from sqlbuild.cursor_algebra.models import AlignedInterval - - -def clamp(*, interval: AlignedInterval, bounds: AlignedInterval) -> AlignedInterval | None: - """Intersect two compatible half-open intervals.""" - - return clamp_interval(interval=interval, bounds=bounds) diff --git a/src/sqlbuild/cursor_algebra/main/compare.py b/src/sqlbuild/cursor_algebra/main/compare.py deleted file mode 100644 index 3b78470c1c..0000000000 --- a/src/sqlbuild/cursor_algebra/main/compare.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Compare typed cursor scalars.""" - -from sqlbuild.cursor_algebra._helpers.comparison import compare_scalars -from sqlbuild.cursor_algebra.types import CursorScalar - - -def compare(*, left: CursorScalar, right: CursorScalar) -> int: - """Compare compatible cursor scalar values.""" - - return compare_scalars(left=left, right=right) diff --git a/src/sqlbuild/cursor_algebra/main/cursor_sort_key.py b/src/sqlbuild/cursor_algebra/main/cursor_sort_key.py deleted file mode 100644 index a393854f80..0000000000 --- a/src/sqlbuild/cursor_algebra/main/cursor_sort_key.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Build a sortable raw cursor key.""" - -from datetime import datetime - -from sqlbuild.cursor_algebra._helpers.comparison import as_scalar, comparison_value - - -def cursor_sort_key(*, raw: object, cursor_type: str) -> datetime | int: - """Return the canonical sortable key for one raw cursor bound.""" - - return comparison_value(value=as_scalar(raw=raw, cursor_type=cursor_type)) diff --git a/src/sqlbuild/cursor_algebra/main/exclusive_end_from_observed_max.py b/src/sqlbuild/cursor_algebra/main/exclusive_end_from_observed_max.py deleted file mode 100644 index c6373aa84b..0000000000 --- a/src/sqlbuild/cursor_algebra/main/exclusive_end_from_observed_max.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Derive availability from an observed cursor maximum.""" - -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra._helpers.intervals import observed_partition_value -from sqlbuild.cursor_algebra.types import CursorScalar - - -def exclusive_end_from_observed_max( - *, value: CursorScalar, grain: CursorGrain | None -) -> CursorScalar: - """Convert an observed maximum to its partition's exclusive end.""" - - return observed_partition_value(value=value, grain=grain).end diff --git a/src/sqlbuild/cursor_algebra/main/exclusive_to_inclusive.py b/src/sqlbuild/cursor_algebra/main/exclusive_to_inclusive.py deleted file mode 100644 index 531e5a1361..0000000000 --- a/src/sqlbuild/cursor_algebra/main/exclusive_to_inclusive.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Convert an exclusive cursor bound to an inclusive bound.""" - -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra._helpers.intervals import shift_inclusive_bound -from sqlbuild.cursor_algebra.types import CursorScalar - - -def exclusive_to_inclusive(*, value: CursorScalar, grain: CursorGrain | None) -> CursorScalar: - """Move an exclusive bound back by one configured cursor unit.""" - - return shift_inclusive_bound(value=value, grain=grain, direction=-1) diff --git a/src/sqlbuild/cursor_algebra/main/floor_to_grain.py b/src/sqlbuild/cursor_algebra/main/floor_to_grain.py deleted file mode 100644 index 71e40a5d86..0000000000 --- a/src/sqlbuild/cursor_algebra/main/floor_to_grain.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Floor a typed cursor scalar.""" - -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra._helpers.arithmetic import floor_scalar -from sqlbuild.cursor_algebra.types import CursorScalar - - -def floor_to_grain(*, value: CursorScalar, grain: CursorGrain) -> CursorScalar: - """Return the aligned boundary at or before a cursor value.""" - - return floor_scalar(value=value, grain=grain) diff --git a/src/sqlbuild/cursor_algebra/main/inclusive_to_exclusive.py b/src/sqlbuild/cursor_algebra/main/inclusive_to_exclusive.py deleted file mode 100644 index dbc883ce01..0000000000 --- a/src/sqlbuild/cursor_algebra/main/inclusive_to_exclusive.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Convert an inclusive cursor bound to an exclusive bound.""" - -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra._helpers.intervals import shift_inclusive_bound -from sqlbuild.cursor_algebra.types import CursorScalar - - -def inclusive_to_exclusive(*, value: CursorScalar, grain: CursorGrain | None) -> CursorScalar: - """Advance an operator-declared inclusive bound by one cursor unit.""" - - return shift_inclusive_bound(value=value, grain=grain, direction=1) diff --git a/src/sqlbuild/cursor_algebra/main/max_bound.py b/src/sqlbuild/cursor_algebra/main/max_bound.py deleted file mode 100644 index 9ff3d9c008..0000000000 --- a/src/sqlbuild/cursor_algebra/main/max_bound.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Select the maximum typed cursor bound.""" - -from sqlbuild.cursor_algebra._helpers.comparison import as_scalar, comparison_value - - -def max_bound[T](*, values: tuple[T, ...] | list[T], cursor_type: str) -> T: - """Select the numerically or temporally largest raw bound.""" - - return max( - values, - key=lambda raw: comparison_value(value=as_scalar(raw=raw, cursor_type=cursor_type)), - ) diff --git a/src/sqlbuild/cursor_algebra/main/merge.py b/src/sqlbuild/cursor_algebra/main/merge.py deleted file mode 100644 index 99ecf43741..0000000000 --- a/src/sqlbuild/cursor_algebra/main/merge.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Merge typed cursor intervals.""" - -from sqlbuild.cursor_algebra._helpers.intervals import merge_interval_values -from sqlbuild.cursor_algebra.models import AlignedInterval - - -def merge(*, intervals: tuple[AlignedInterval, ...]) -> tuple[AlignedInterval, ...]: - """Merge overlapping or adjacent canonical intervals.""" - - return merge_interval_values(intervals=intervals) diff --git a/src/sqlbuild/cursor_algebra/main/min_bound.py b/src/sqlbuild/cursor_algebra/main/min_bound.py deleted file mode 100644 index c3ab324ea4..0000000000 --- a/src/sqlbuild/cursor_algebra/main/min_bound.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Select the minimum typed cursor bound.""" - -from sqlbuild.cursor_algebra._helpers.comparison import as_scalar, comparison_value - - -def min_bound[T](*, values: tuple[T, ...] | list[T], cursor_type: str) -> T: - """Select the numerically or temporally smallest raw bound.""" - - return min( - values, - key=lambda raw: comparison_value(value=as_scalar(raw=raw, cursor_type=cursor_type)), - ) diff --git a/src/sqlbuild/cursor_algebra/main/next_boundary.py b/src/sqlbuild/cursor_algebra/main/next_boundary.py deleted file mode 100644 index 1266c72c12..0000000000 --- a/src/sqlbuild/cursor_algebra/main/next_boundary.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Advance a typed cursor boundary.""" - -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra._helpers.arithmetic import advance_scalar -from sqlbuild.cursor_algebra.types import CursorScalar - - -def next_boundary(*, value: CursorScalar, grain: CursorGrain) -> CursorScalar: - """Return the boundary one grain after an aligned cursor value.""" - - return advance_scalar(value=value, grain=grain) diff --git a/src/sqlbuild/cursor_algebra/main/observed_partition.py b/src/sqlbuild/cursor_algebra/main/observed_partition.py deleted file mode 100644 index cc8a0c5529..0000000000 --- a/src/sqlbuild/cursor_algebra/main/observed_partition.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Build the partition containing an observed cursor value.""" - -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra._helpers.intervals import observed_partition_value -from sqlbuild.cursor_algebra.models import AlignedInterval -from sqlbuild.cursor_algebra.types import CursorScalar - - -def observed_partition(*, value: CursorScalar, grain: CursorGrain | None) -> AlignedInterval: - """Return the canonical partition containing an observed physical value.""" - - return observed_partition_value(value=value, grain=grain) diff --git a/src/sqlbuild/cursor_algebra/main/parse.py b/src/sqlbuild/cursor_algebra/main/parse.py deleted file mode 100644 index f6acfceea4..0000000000 --- a/src/sqlbuild/cursor_algebra/main/parse.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Parse a typed cursor scalar.""" - -from sqlbuild.cursor_algebra._helpers.parsing import parse_scalar -from sqlbuild.cursor_algebra.types import CursorScalar - - -def parse(*, raw: object, cursor_type: str) -> CursorScalar: - """Parse one cursor scalar, preserving date-ness and timestamp offsets.""" - - return parse_scalar(raw=raw, cursor_type=cursor_type) diff --git a/src/sqlbuild/cursor_algebra/main/render.py b/src/sqlbuild/cursor_algebra/main/render.py deleted file mode 100644 index a3add5d125..0000000000 --- a/src/sqlbuild/cursor_algebra/main/render.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Render a typed cursor scalar.""" - -from sqlbuild.cursor_algebra._helpers.parsing import render_scalar -from sqlbuild.cursor_algebra.types import CursorScalar - - -def render(*, value: CursorScalar) -> str: - """Render a scalar in the existing cursor output format.""" - - return render_scalar(value=value) diff --git a/src/sqlbuild/cursor_algebra/main/sentinel_from_token.py b/src/sqlbuild/cursor_algebra/main/sentinel_from_token.py deleted file mode 100644 index 12ae25c219..0000000000 --- a/src/sqlbuild/cursor_algebra/main/sentinel_from_token.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Parse a typed cursor sentinel.""" - -from sqlbuild.cursor_algebra.types import BoundSentinel - - -def sentinel_from_token(*, token: str) -> BoundSentinel | None: - """Parse a SQL token as a typed sentinel when recognized.""" - - try: - return BoundSentinel(token) - except ValueError: - return None diff --git a/src/sqlbuild/cursor_algebra/main/sentinel_to_token.py b/src/sqlbuild/cursor_algebra/main/sentinel_to_token.py deleted file mode 100644 index 161fa1a7ea..0000000000 --- a/src/sqlbuild/cursor_algebra/main/sentinel_to_token.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Serialize a typed cursor bound.""" - -from sqlbuild.cursor_algebra.main.render import render -from sqlbuild.cursor_algebra.types import Bound, BoundSentinel - - -def sentinel_to_token(*, sentinel: Bound) -> str: - """Serialize a scalar or sentinel while retaining the phase-1 sentinel API.""" - - return sentinel.value if isinstance(sentinel, BoundSentinel) else render(value=sentinel) diff --git a/src/sqlbuild/cursor_algebra/main/split_into_batches.py b/src/sqlbuild/cursor_algebra/main/split_into_batches.py deleted file mode 100644 index 621fc2056b..0000000000 --- a/src/sqlbuild/cursor_algebra/main/split_into_batches.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Split a typed cursor interval into batches.""" - -from sqlbuild.cursor_algebra._helpers.intervals import split_interval -from sqlbuild.cursor_algebra.models import AlignedInterval - - -def split_into_batches(*, interval: AlignedInterval, step: int) -> tuple[AlignedInterval, ...]: - """Split an interval into batches of a positive number of grain units.""" - - return split_interval(interval=interval, step=step) diff --git a/src/sqlbuild/cursor_algebra/main/try_parse.py b/src/sqlbuild/cursor_algebra/main/try_parse.py deleted file mode 100644 index 2eb9090fb0..0000000000 --- a/src/sqlbuild/cursor_algebra/main/try_parse.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Tolerantly parse a typed cursor scalar.""" - -from sqlbuild.cursor_algebra._helpers.parsing import parse_scalar -from sqlbuild.cursor_algebra.types import CursorScalar - - -def try_parse(*, raw: object, cursor_type: str) -> CursorScalar | None: - """Parse discovery values which may be DATE or TIMESTAMP values.""" - - try: - return parse_scalar(raw=raw, cursor_type=cursor_type) - except (TypeError, ValueError): - return None diff --git a/src/sqlbuild/cursor_algebra/models.py b/src/sqlbuild/cursor_algebra/models.py index f158c16652..02ec9e9332 100644 --- a/src/sqlbuild/cursor_algebra/models.py +++ b/src/sqlbuild/cursor_algebra/models.py @@ -2,12 +2,15 @@ from __future__ import annotations -import calendar -import re from dataclasses import dataclass, field -from datetime import date, datetime, timedelta +from datetime import date, datetime from typing import ClassVar +from sqlbuild._native import ( + parse_cursor_duration, + shift_by_cursor_duration, + validate_cursor_interval, +) from sqlbuild.compiler.planner.types import CursorGrain from sqlbuild.cursor_algebra.constants import ( DURATION_DAY_UNIT, @@ -17,7 +20,6 @@ DURATION_SECOND_UNIT, DURATION_YEAR_UNIT, ) -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError @dataclass(frozen=True, kw_only=True) @@ -51,28 +53,13 @@ class AlignedInterval: grain: CursorGrain | None def __post_init__(self) -> None: - from sqlbuild.cursor_algebra.main.compare import compare - from sqlbuild.cursor_algebra.main.floor_to_grain import floor_to_grain - - if compare(left=self.start, right=self.end) >= 0: - raise CursorAlgebraError("cursor interval start must be less than end") - if self.grain is None: - if not isinstance(self.start, IntegerValue) or not isinstance(self.end, IntegerValue): - raise CursorAlgebraError("temporal cursor intervals require a grain") - return - if floor_to_grain(value=self.start, grain=self.grain) != self.start: - raise CursorAlgebraError("cursor interval start must be grain-aligned") - if floor_to_grain(value=self.end, grain=self.grain) != self.end: - raise CursorAlgebraError("cursor interval end must be grain-aligned") + validate_cursor_interval(start=self.start, end=self.end, grain=self.grain) @dataclass(frozen=True) class Duration: """A calendar-aware duration (years/months plus fixed time units).""" - _PATTERN: ClassVar[re.Pattern[str]] = re.compile( - r"^(?:(\d+)y)?(?:(\d+)mo)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$" - ) _MONTHS_PER_YEAR: ClassVar[int] = 12 _SECONDS_PER_DAY: ClassVar[int] = 86_400 _SECONDS_PER_HOUR: ClassVar[int] = 3_600 @@ -89,20 +76,13 @@ class Duration: def parse(cls, value: str) -> Duration | None: """Parse a string like '1mo', '1y6mo', '30d', or '6h' into a duration.""" - match: re.Match[str] | None = cls._PATTERN.match(value) - if match is None: + parts: tuple[int, int, int, int, int, int] | None = parse_cursor_duration(value) + if parts is None: return None - duration: Duration = cls( - years=int(match.group(1) or 0), - months=int(match.group(2) or 0), - days=int(match.group(3) or 0), - hours=int(match.group(4) or 0), - minutes=int(match.group(5) or 0), - seconds=int(match.group(6) or 0), + years, months, days, hours, minutes, seconds = parts + return cls( + years=years, months=months, days=days, hours=hours, minutes=minutes, seconds=seconds ) - if duration.is_zero: - return None - return duration @property def is_zero(self) -> bool: @@ -152,33 +132,12 @@ def total_months(self) -> int: return self.years * self._MONTHS_PER_YEAR + self.months - @property - def _fixed_timedelta(self) -> timedelta: - return timedelta( - days=self.days, hours=self.hours, minutes=self.minutes, seconds=self.seconds - ) - def add_to(self, moment: datetime) -> datetime: """Return the moment advanced by this duration.""" - return ( - self._shift_months(moment=moment, months_delta=self.total_months) - + self._fixed_timedelta - ) + return shift_by_cursor_duration(moment=moment, duration=self, direction=1) def subtract_from(self, moment: datetime) -> datetime: """Return the moment moved back by this duration.""" - return ( - self._shift_months(moment=moment, months_delta=-self.total_months) - - self._fixed_timedelta - ) - - def _shift_months(self, *, moment: datetime, months_delta: int) -> datetime: - if months_delta == 0: - return moment - total: int = (moment.year * self._MONTHS_PER_YEAR + (moment.month - 1)) + months_delta - year: int = total // self._MONTHS_PER_YEAR - month: int = total % self._MONTHS_PER_YEAR + 1 - day: int = min(moment.day, calendar.monthrange(year, month)[1]) - return moment.replace(year=year, month=month, day=day) + return shift_by_cursor_duration(moment=moment, duration=self, direction=-1) diff --git a/src/sqlbuild/executor/clone/_helpers/canonical_progress.py b/src/sqlbuild/executor/clone/_helpers/canonical_progress.py index 4ab17c1cc3..23ebbe7008 100644 --- a/src/sqlbuild/executor/clone/_helpers/canonical_progress.py +++ b/src/sqlbuild/executor/clone/_helpers/canonical_progress.py @@ -5,7 +5,7 @@ from collections.abc import Iterator from contextlib import contextmanager -from sqlbuild.cli.progress.classes.native_progress_projector import ( +from sqlbuild._native import ( NativeProgressProjector, current_native_progress_projector, ) diff --git a/src/sqlbuild/executor/diff/_helpers/bounds.py b/src/sqlbuild/executor/diff/_helpers/bounds.py index 3bc9404a4c..50fd17346b 100644 --- a/src/sqlbuild/executor/diff/_helpers/bounds.py +++ b/src/sqlbuild/executor/diff/_helpers/bounds.py @@ -5,11 +5,11 @@ from datetime import UTC, datetime from typing import Any +from sqlbuild._native import get_config_str from sqlbuild.adapter.contract.models import CursorValue from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.compiler.planner.models import Duration from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.spec.contracts.main.get_config_str import get_config_str def resolve_bounded_cursors( diff --git a/src/sqlbuild/executor/diff/_helpers/size_guard.py b/src/sqlbuild/executor/diff/_helpers/size_guard.py index 64f1eeb404..56537b44bd 100644 --- a/src/sqlbuild/executor/diff/_helpers/size_guard.py +++ b/src/sqlbuild/executor/diff/_helpers/size_guard.py @@ -4,12 +4,12 @@ from typing import Any +from sqlbuild._native import get_config_str from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationRowCountEstimate from sqlbuild.executor.diff._helpers.selection import qualified_name from sqlbuild.executor.diff.exceptions import FullDiffSizeGuardError from sqlbuild.executor.diff.models import FullDiffModelSize, FullDiffSideSize, FullDiffSizeLimits -from sqlbuild.spec.contracts.main.get_config_str import get_config_str def enforce_full_diff_size_limits( diff --git a/src/sqlbuild/executor/load/_helpers/cursors.py b/src/sqlbuild/executor/load/_helpers/cursors.py index 4f997b2eca..4f91b530d7 100644 --- a/src/sqlbuild/executor/load/_helpers/cursors.py +++ b/src/sqlbuild/executor/load/_helpers/cursors.py @@ -6,11 +6,11 @@ from decimal import Decimal from typing import Any +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import CursorKind -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index edea222d55..1ecac28fe1 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -15,12 +15,12 @@ from pathlib import Path from typing import Any, ClassVar, Protocol -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.cli.progress.classes.native_progress_projector import ( +from sqlbuild._native import ( NativeProgressProjector, current_native_progress_projector, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.main.scenarios.scenario import build_scenario_plan diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 1c6fc03da1..c7061b769f 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, replace from typing import Any +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -20,7 +21,6 @@ from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry from sqlbuild.compiler.planner.types import IncrementalStrategy, OnSchemaChange -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 397b011591..6458352071 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -12,6 +12,18 @@ from typing import Any from sqlbuild._native import compute_query_hash +from sqlbuild._native import cursor_cap_from_end as cap_from_end +from sqlbuild._native import cursor_cap_from_start as cap_from_start +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import ( + cursor_exclusive_end_from_observed_max as exclusive_end_from_observed_max, +) +from sqlbuild._native import cursor_floor_to_grain as floor_to_grain +from sqlbuild._native import cursor_next_boundary as next_boundary +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import cursor_split_into_batches as split_into_batches from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -50,18 +62,6 @@ MicrobatchStrategy, OnSchemaChange, ) -from sqlbuild.cursor_algebra.main.cap_from_end import cap_from_end -from sqlbuild.cursor_algebra.main.cap_from_start import cap_from_start -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.exclusive_end_from_observed_max import ( - exclusive_end_from_observed_max, -) -from sqlbuild.cursor_algebra.main.floor_to_grain import floor_to_grain -from sqlbuild.cursor_algebra.main.next_boundary import next_boundary -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token -from sqlbuild.cursor_algebra.main.split_into_batches import split_into_batches from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index ee9a4512c1..e8503ef03f 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -8,6 +8,20 @@ from functools import partial from typing import Any +from sqlbuild._native import cursor_clamp as clamp +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import ( + cursor_exclusive_end_from_observed_max as exclusive_end_from_observed_max, +) +from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive +from sqlbuild._native import cursor_floor_to_grain as floor_to_grain +from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive +from sqlbuild._native import cursor_max_bound as max_bound +from sqlbuild._native import cursor_min_bound as min_bound +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render +from sqlbuild._native import cursor_sentinel_from_token as sentinel_from_token +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics @@ -43,20 +57,6 @@ MicrobatchStrategy, ) from sqlbuild.cursor_algebra.constants import GRAIN_ORDER -from sqlbuild.cursor_algebra.main.clamp import clamp -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.exclusive_end_from_observed_max import ( - exclusive_end_from_observed_max, -) -from sqlbuild.cursor_algebra.main.exclusive_to_inclusive import exclusive_to_inclusive -from sqlbuild.cursor_algebra.main.floor_to_grain import floor_to_grain -from sqlbuild.cursor_algebra.main.inclusive_to_exclusive import inclusive_to_exclusive -from sqlbuild.cursor_algebra.main.max_bound import max_bound -from sqlbuild.cursor_algebra.main.min_bound import min_bound -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render -from sqlbuild.cursor_algebra.main.sentinel_from_token import sentinel_from_token -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.cursor_algebra.models import ( AlignedInterval, DateValue, diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 3f1c9488c6..1ea931c07e 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -8,6 +8,8 @@ from datetime import datetime from typing import Any, cast +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_parse as parse from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult @@ -24,8 +26,6 @@ from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.parse import parse from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/src/sqlbuild/integrations/dagster/_helpers/invocation.py b/src/sqlbuild/integrations/dagster/_helpers/invocation.py index 5b9ca60432..1a218f9ce1 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/invocation.py +++ b/src/sqlbuild/integrations/dagster/_helpers/invocation.py @@ -6,12 +6,11 @@ import tempfile from collections.abc import Mapping from concurrent.futures import Future, ThreadPoolExecutor -from dataclasses import asdict from enum import Enum from pathlib import Path from typing import IO, TYPE_CHECKING, Any, TextIO -from sqlbuild.cli.output.models import IntegrationAssetResult, IntegrationResultEnvelope +from sqlbuild._native import IntegrationAssetResult, IntegrationResultEnvelope from sqlbuild.integrations.dagster._helpers.translation import ( is_dagster_asset_enabled, is_dagster_check_enabled, @@ -549,12 +548,12 @@ def _build_results_from_integration_result( loader_result = _loader_result_for_source_payload( dag=dag, source_node=node, - payload_asset=asdict(asset), + payload_asset=asset.as_dict(), loader_name=asset.loader, ) asset_nodes: dict[str, tuple[Mapping[str, Any], Mapping[str, Any] | None]] = {} if node is not None: - asset_nodes[str(node.get("id"))] = (node, asdict(asset)) + asset_nodes[str(node.get("id"))] = (node, asset.as_dict()) if loader_result is not None: loader_id, loader_payload = loader_result loader_node: Mapping[str, Any] | None = nodes_by_id.get(loader_id) @@ -608,7 +607,7 @@ def _build_results_from_integration_result( nodes_by_id=nodes_by_id, nodes_by_name=nodes_by_name, check={ - **asdict(check), + **check.as_dict(), "event_id": envelope.event_id, "resource_attempt_id": envelope.resource_attempt_id, "resource_id": envelope.resource_id, diff --git a/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_invocation.py b/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_invocation.py index f84783c878..c7361ba805 100644 --- a/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_invocation.py +++ b/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_invocation.py @@ -12,8 +12,7 @@ from pathlib import Path from typing import IO, Any -from sqlbuild.cli.output.constants import INTEGRATION_MICROBATCH_KEYS -from sqlbuild.cli.output.models import IntegrationAssetResult, IntegrationResultEnvelope +from sqlbuild._native import IntegrationAssetResult, IntegrationResultEnvelope from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster._helpers.invocation import ( _build_results_for_selected_assets, @@ -31,29 +30,6 @@ from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError _MAX_DIAGNOSTIC_CHARS: int = 4_000 -_OPTIONAL_ASSET_ENRICHMENT_FIELDS: frozenset[str] = frozenset( - { - "action", - "failed_phase", - "future_cursor_safety", - "maximum_start_safety", - } -) -_OPTIONAL_MICROBATCH_ENUM_FIELDS: frozenset[str] = frozenset( - { - "action", - "replay_requirement_state", - "run_type", - "unaccounted_partition_policy", - } -) -_OPTIONAL_CHECK_ENRICHMENT_FIELDS: frozenset[str] = frozenset( - { - "attachment_kind", - "run_scope_phase", - "severity", - } -) def _diagnostic_tail(value: str) -> str: @@ -408,7 +384,7 @@ def _results_from_live_event_line( envelope: IntegrationResultEnvelope = IntegrationResultEnvelope.from_json(line) except ObservabilityValidationError: try: - envelope = _integration_result_without_optional_enrichment(line) + envelope = IntegrationResultEnvelope.from_json_without_optional_enrichment(line) except ObservabilityValidationError: self._warn_invalid_integration_record() return @@ -484,46 +460,3 @@ def _log_live_asset_failure( extra=metadata, ) return logged_failed_assets - - -def _integration_result_without_optional_enrichment(line: str) -> IntegrationResultEnvelope: - """Retry a valid envelope after dropping forward-compatible optional enrichment.""" - - try: - payload: Any = json.loads(line) - except json.JSONDecodeError as error: - raise ObservabilityValidationError("malformed integration result JSON") from error - if not isinstance(payload, dict): - raise ObservabilityValidationError("integration result must be a JSON object") - asset: object = payload.get("asset") - if isinstance(asset, dict): - stripped_asset: dict[str, object] = { - key: value - for key, value in asset.items() - if key not in _OPTIONAL_ASSET_ENRICHMENT_FIELDS - } - microbatch: object = asset.get("microbatch") - if isinstance(microbatch, dict): - stripped_asset["microbatch"] = { - key: value - for key, value in microbatch.items() - if key in INTEGRATION_MICROBATCH_KEYS - and key not in _OPTIONAL_MICROBATCH_ENUM_FIELDS - } - payload["asset"] = stripped_asset - checks: object = payload.get("checks") - if isinstance(checks, list): - stripped_checks: list[object] = [] - for check in checks: - stripped_check: object = ( - { - key: value - for key, value in check.items() - if key not in _OPTIONAL_CHECK_ENRICHMENT_FIELDS - } - if isinstance(check, dict) - else check - ) - stripped_checks.append(stripped_check) - payload["checks"] = stripped_checks - return IntegrationResultEnvelope.from_json(json.dumps(payload)) diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py index b641163643..8c131df15f 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import TextIO, cast +from sqlbuild._native import format_summary_footer from sqlbuild.integrations.dbt._helpers.graph.core import ( dbt_model_graph_key, expand_combined_downstream, @@ -48,7 +49,6 @@ ) from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.summary_footer import format_summary_footer _COMMAND_AND_SUBCOMMAND_ARGUMENT_COUNT: int = 2 _DBT_LS_EXCLUDED_EXECUTION_FLAGS: frozenset[str] = frozenset({"--full-refresh", "--fail-fast"}) diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py index 94e1dece9a..9173ead186 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py @@ -4,13 +4,12 @@ from dataclasses import replace +from sqlbuild._native import ConnectionProgressReporter, apply_command_warehouse from sqlbuild.cli.commands.main.dbt.dbt_sqlbuild_work import execute_dbt_sqlbuild_work -from sqlbuild.cli.commands.main.execution.command_warehouse import apply_command_warehouse from sqlbuild.cli.commands.main.execution.connection_progress import ( build_connection_progress_reporter, ) from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import PlanOutput diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py index f392acd97c..dd3a7ac299 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py @@ -9,10 +9,10 @@ from typing import Any from sqlbuild._native import ( + apply_command_warehouse, resolve_model_deferral_inputs, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.main.execution.command_warehouse import apply_command_warehouse from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py b/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py index 0a5371ae24..3ab0487e19 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py +++ b/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py @@ -7,15 +7,13 @@ from collections import defaultdict from collections.abc import Sequence -from sqlbuild._native import format_plan +from sqlbuild._native import append_overflow_line, format_plan, visible_entries from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.integrations.dbt.models import DbtInteropPlan, DbtInteropSelectionResult, DbtLsNode from sqlbuild.integrations.dbt.types import DbtInteropCommand, DbtInteropSkipReason from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.append_overflow_line import append_overflow_line from sqlbuild.presentation.main.surface_header import format_surface_header from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.main.visible_entries import visible_entries from sqlbuild.presentation.models import DisplayOptions _ANSI_ESCAPE_PATTERN: re.Pattern[str] = re.compile(r"\033\[[0-9;]*m") diff --git a/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py b/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py index ecb7be682a..ad3096637d 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py +++ b/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py @@ -12,11 +12,11 @@ from pathlib import Path from typing import TextIO, cast +from sqlbuild._native import format_aligned_name_value from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError from sqlbuild.integrations.dbt.models import DbtNodeExecutionResult, DbtNodeMessage from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.aligned_name_value import format_aligned_name_value from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle _DBT_ERROR_LEVEL: str = "error" diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index 4c7480a36a..9a11a2c204 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -7,7 +7,7 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, external_identifiers_for_headers from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from sqlbuild.compiler.compile.constants import MACRO_TOKEN @@ -25,7 +25,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint._helpers.headers import lint_body_ranges, lint_file_role -from sqlbuild.lint._helpers.native import external_identifiers_for_headers from sqlbuild.lint._helpers.native_sql import check_lint_not_stopped from sqlbuild.lint._helpers.sqlbuild_tokens import ( neutralize_context_interpolation, diff --git a/src/sqlbuild/lint/_helpers/headers.py b/src/sqlbuild/lint/_helpers/headers.py index 9dc2ccb033..f34848ff89 100644 --- a/src/sqlbuild/lint/_helpers/headers.py +++ b/src/sqlbuild/lint/_helpers/headers.py @@ -1,34 +1,20 @@ -"""Scanner that locates DSL header regions inside SQL files.""" +"""Lintable body ranges of SQL files by project role, and externally resolved descriptions.""" from __future__ import annotations -import re -from functools import cache from pathlib import Path, PurePath +from sqlbuild._native import measurement_body_ranges, sql_body_ranges +from sqlbuild.compiler.compile.classes.model_description_resolution import ( + ModelDescriptionResolution, +) from sqlbuild.compiler.compile.constants import HOOK_DIRECTORY_NAME from sqlbuild.compiler.discovery.main.named_declaration_role_kind import ( named_declaration_role_kind, ) from sqlbuild.compiler.scopes.types import DeclarationKind -from sqlbuild.lint.constants import DSL_HEADER_KINDS -from sqlbuild.lint.models import HeaderSpan, LintFileRole - -_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) -_ESCAPE_CHARACTER: str = "\\" -_OPEN_PAREN: str = "(" -_CLOSE_PAREN: str = ")" -_STATEMENT_TERMINATOR: str = ";" -_UNQUOTED_HEADER_TOKEN: re.Pattern[str] = re.compile(r"""[()'"]""") -_QUOTED_HEADER_TOKENS: dict[str, re.Pattern[str]] = { - quote: re.compile(rf"[\\{quote}]") for quote in _QUOTE_CHARACTERS -} - - -def scan_headers(*, contents: str, first_only: bool = False) -> tuple[HeaderSpan, ...]: - """Return all DSL header spans found in the file contents.""" - - return _scan_delimited_regions(contents=contents, kinds=DSL_HEADER_KINDS, first_only=first_only) +from sqlbuild.lint.constants import RULE_DESCRIPTION_PRESENT +from sqlbuild.lint.models import HeaderSpan, LintFileRole, LintViolation def lint_file_role( @@ -47,24 +33,6 @@ def lint_file_role( ) -def measurement_body_ranges(*, contents: str) -> tuple[tuple[int, int], ...]: - """Return authored query ranges inside MEASURE and EVIDENCE blocks.""" - - blocks: tuple[HeaderSpan, ...] = _scan_delimited_regions( - contents=contents, kinds=frozenset({"MEASURE", "EVIDENCE"}) - ) - ranges: list[tuple[int, int]] = [] - for block in blocks: - opening_index: int = contents.find(_OPEN_PAREN, block.start, block.end) - closing_index: int = contents.rfind(_CLOSE_PAREN, opening_index + 1, block.end) - body_start: int = _first_non_whitespace( - contents=contents, start=opening_index + 1, end=closing_index - ) - if body_start < closing_index: - ranges.append((body_start, closing_index)) - return tuple(ranges) - - def lint_body_ranges( *, contents: str, @@ -88,119 +56,20 @@ def lint_body_ranges( return sql_body_ranges(contents=contents, headers=headers) -def _scan_delimited_regions( - *, contents: str, kinds: frozenset[str], first_only: bool = False -) -> tuple[HeaderSpan, ...]: - """Return line-leading parenthesized regions with one of the requested names.""" - - spans: list[HeaderSpan] = [] - covered_until: int = 0 - match: re.Match[str] - for match in _delimited_region_pattern(kinds).finditer(contents): - kind: str | None = match.group("kind") - if kind is None: - continue - keyword_start: int = match.start("kind") - if keyword_start < covered_until: - continue - span: HeaderSpan | None = _match_header_span( - contents=contents, - kind=kind, - keyword_start=keyword_start, - body_start=match.end("kind"), - ) - if span is not None: - spans.append(span) - covered_until = span.end - if first_only: - break - return tuple(spans) - +def drop_resolved_description_faults( + *, violations: list[LintViolation], project_dir: Path +) -> list[LintViolation]: + """Drop `description-present` faults for models described outside their header.""" -@cache -def _delimited_region_pattern(kinds: frozenset[str]) -> re.Pattern[str]: - """Build a C-level scanner that skips comments and SQL strings.""" - - kind_alternatives: str = "|".join(re.escape(kind) for kind in sorted(kinds)) - return re.compile( - rf"--[^\n]*(?:\n|\Z)" - rf"|/\*[\s\S]*?(?:\*/|\Z)" - rf"|'(?:''|[^'])*(?:'|\Z)" - rf'|"(?:""|[^"])*(?:"|\Z)' - rf"|(?m:^[^\S\r\n]*(?P{kind_alternatives})\b)" + if not any(violation.code == RULE_DESCRIPTION_PRESENT for violation in violations): + return violations + described: frozenset[Path] = frozenset( + path.resolve() + for path in ModelDescriptionResolution.externally_described_models(project_dir=project_dir) ) - - -def sql_body_ranges( - *, contents: str, headers: tuple[HeaderSpan, ...] -) -> tuple[tuple[int, int], ...]: - """Return non-empty SQL body ranges between consecutive headers.""" - - ranges: list[tuple[int, int]] = [] - cursor: int = 0 - header: HeaderSpan - for header in headers: - body_start: int = _first_non_whitespace(contents=contents, start=cursor, end=header.start) - if body_start < header.start: - ranges.append((body_start, header.start)) - cursor = header.end - tail_start: int = _first_non_whitespace(contents=contents, start=cursor, end=len(contents)) - if tail_start < len(contents): - ranges.append((tail_start, len(contents))) - return tuple(ranges) - - -def _match_header_span( - *, contents: str, kind: str, keyword_start: int, body_start: int -) -> HeaderSpan | None: - index: int = _skip_whitespace(contents=contents, index=body_start) - if index >= len(contents) or contents[index] != _OPEN_PAREN: - return None - depth: int = 0 - in_quote: str | None = None - match: re.Match[str] | None - while True: - match = ( - _UNQUOTED_HEADER_TOKEN if in_quote is None else _QUOTED_HEADER_TOKENS[in_quote] - ).search(contents, index) - if match is None: - return None - index = match.start() - character: str = contents[index] - if in_quote is not None: - if character == _ESCAPE_CHARACTER: - index += 2 - continue - in_quote = None - elif character in _QUOTE_CHARACTERS: - in_quote = character - elif character == _OPEN_PAREN: - depth += 1 - else: - depth -= 1 - if depth == 0: - terminator_index: int = _skip_whitespace(contents=contents, index=index + 1) - span_end: int = index + 1 - if ( - terminator_index < len(contents) - and contents[terminator_index] == _STATEMENT_TERMINATOR - ): - span_end = terminator_index + 1 - return HeaderSpan(kind=kind, start=keyword_start, end=span_end) - index += 1 - - -def _skip_whitespace(*, contents: str, index: int) -> int: - length: int = len(contents) - while index < length and contents[index].isspace(): - index += 1 - return index - - -def _first_non_whitespace(*, contents: str, start: int, end: int) -> int: - index: int = start - while index < end: - if not contents[index].isspace(): - return index - index += 1 - return end + return [ + violation + for violation in violations + if violation.code != RULE_DESCRIPTION_PRESENT + or violation.file_path.resolve() not in described + ] diff --git a/src/sqlbuild/lint/_helpers/native.py b/src/sqlbuild/lint/_helpers/native.py deleted file mode 100644 index 6ec46d75b9..0000000000 --- a/src/sqlbuild/lint/_helpers/native.py +++ /dev/null @@ -1,864 +0,0 @@ -"""Native DSL header lint rules and formatting fixes.""" - -from __future__ import annotations - -import re -import textwrap -from bisect import bisect_right -from dataclasses import dataclass -from pathlib import Path - -from sqlbuild._native import parse_header_values, prepare_model_header_tokens -from sqlbuild.compiler.compile.classes.model_description_resolution import ( - ModelDescriptionResolution, -) -from sqlbuild.compiler.discovery.exceptions import DiscoveryError, ModelSqlParseError -from sqlbuild.lint.constants import ( - CARRIAGE_RETURN_LINE_FEED, - CLOSING_PAREN_CHARACTER, - DESCRIPTION_HEADER_KINDS, - DESCRIPTION_REQUIRED_HEADER_KINDS, - HEADER_KIND_FUNCTION, - HEADER_KIND_MODEL, - IDENTIFIER_SEPARATOR_CHARACTER, - LINE_FEED, - LINT_ENGINE_SQLBUILD, - RULE_DESCRIPTION_LENGTH, - RULE_DESCRIPTION_PRESENT, - RULE_HEADER_PARSE, - RULE_HEADER_WHITESPACE, - RULE_LEADING_COMMENT_DESCRIPTION, - VIOLATION_SEVERITY_FAULT, - VIOLATION_SEVERITY_WARNING, -) -from sqlbuild.lint.models import HeaderSpan, LintConfig, LintViolation - -_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) -_ESCAPE_CHARACTER: str = "\\" -_NEWLINE: str = "\n" -_BLOCK_COMMENT_START: str = "/*" -_BLOCK_COMMENT_END: str = "*/" -_LINE_COMMENT_PREFIX: str = "--" -_DESCRIPTION_KEY: str = "description" -_DESCRIPTION_SENTINEL_PATH: str = "" -_HEADER_INDENT: str = " " -_UNICODE_MAX_CODEPOINT: int = 0x10FFFF -_UNICODE_SURROGATE_START: int = 0xD800 -_UNICODE_SURROGATE_END: int = 0xDFFF -_HEXADECIMAL_CHARACTERS: frozenset[str] = frozenset("0123456789abcdefABCDEF") - - -_AUDITS_KEY: re.Pattern[str] = re.compile(r"\baudits\s*\[") -_OPENERS: dict[str, str] = {"(": ")", "[": "]"} -_AUDIT_LIST_OPENER: str = "[" -_HEADER_ITEM_SEPARATOR: str = "," -_AUDITS_KEYWORD: str = "audits" -_AUDIT_WRAP_MAX_PASSES: int = 64 - - -@dataclass(frozen=True) -class _Group: - """One bracket group inside a header audit list.""" - - open_index: int - close_index: int - depth: int - commas: tuple[int, ...] - - -@dataclass(frozen=True) -class _RelocationOutcome: - """Result of relocating a leading comment into the model description.""" - - contents: str - faults: tuple[LintViolation, ...] - - -def lint_native_headers( - *, - contents: str, - file_path: Path, - headers: tuple[HeaderSpan, ...], - config: LintConfig, -) -> tuple[LintViolation, ...]: - """Run all native header rules and return their violations.""" - - violations: list[LintViolation] = [] - line_starts: tuple[int, ...] = source_line_starts(contents) - header: HeaderSpan - for header in headers: - violations.extend( - _lint_header_values( - contents=contents, - file_path=file_path, - header=header, - config=config, - line_starts=line_starts, - ) - ) - violations.extend( - _lint_header_whitespace( - contents=contents, - file_path=file_path, - header=header, - line_starts=line_starts, - ) - ) - return tuple(violations) - - -def prepare_native_header_cache(*, files: dict[Path, str]) -> None: - """Batch parse formatter-owned headers before per-file formatting and linting.""" - - headers: list[str] = [] - from sqlbuild.lint._helpers.headers import scan_headers - - for contents in files.values(): - headers.extend( - _inner_header_text(header_text=contents[header.start : header.end]) - for header in scan_headers(contents=contents) - ) - prepare_model_header_tokens(headers) - - -def safe_format_files( - *, files: dict[Path, str], config: LintConfig -) -> tuple[dict[Path, str], dict[Path, str]]: - """Return parse-safe files together with every authored newline convention.""" - - safe_files: dict[Path, str] = { - file_path: contents - for file_path, contents in files.items() - if not _header_parse_faults(file_path=file_path, contents=contents, config=config) - } - newline_by_path: dict[Path, str] = { - file_path: CARRIAGE_RETURN_LINE_FEED if CARRIAGE_RETURN_LINE_FEED in contents else LINE_FEED - for file_path, contents in files.items() - } - return safe_files, newline_by_path - - -def reject_unparseable_header_rewrites( - *, - updated: dict[Path, str], - config: LintConfig, - faults: list[LintViolation], -) -> tuple[dict[Path, str], list[LintViolation]]: - """Discard every file rewrite whose resulting header cannot be parsed.""" - - accepted: dict[Path, str] = dict(updated) - for file_path, contents in tuple(updated.items()): - parse_faults: tuple[LintViolation, ...] = _header_parse_faults( - file_path=file_path, - contents=contents, - config=config, - ) - if not parse_faults: - continue - accepted.pop(file_path, None) - faults.extend(parse_faults) - return accepted, faults - - -def _header_parse_faults( - *, file_path: Path, contents: str, config: LintConfig -) -> tuple[LintViolation, ...]: - from sqlbuild.lint._helpers.headers import scan_headers - - headers: tuple[HeaderSpan, ...] = scan_headers(contents=contents) - return tuple( - violation - for violation in lint_native_headers( - contents=contents, - file_path=file_path, - headers=headers, - config=config, - ) - if violation.code == RULE_HEADER_PARSE - ) - - -def format_native_headers( - *, - contents: str, - file_path: Path, - config: LintConfig, -) -> tuple[str, tuple[LintViolation, ...]]: - """Apply native header autofixes and return rewritten contents plus faults.""" - - from sqlbuild.lint._helpers.headers import scan_headers - - headers: tuple[HeaderSpan, ...] = scan_headers(contents=contents) - updated: str = _format_header_whitespace(contents=contents, headers=headers) - relocation: _RelocationOutcome = _relocate_leading_comment( - contents=updated, - file_path=file_path, - config=config, - headers=headers, - ) - relocated_headers: tuple[HeaderSpan, ...] = scan_headers(contents=relocation.contents) - wrapped: str = _format_description_wrapping( - contents=relocation.contents, - headers=relocated_headers, - config=config, - ) - audits_wrapped: str = _wrap_header_audits( - contents=wrapped, - headers=scan_headers(contents=wrapped), - line_width=config.line_width, - ) - return audits_wrapped, relocation.faults - - -def _header_values_or_unique(*, kind: str, header_text: str) -> object: - """Parse header values; an unparseable header compares unequal to every other result.""" - - try: - return _parse_header_values(kind=kind, header_text=header_text) - except ModelSqlParseError: - return object() - - -def _lint_header_values( - *, - contents: str, - file_path: Path, - header: HeaderSpan, - config: LintConfig, - line_starts: tuple[int, ...], -) -> tuple[LintViolation, ...]: - header_text: str = contents[header.start : header.end] - try: - values: dict[str, object] = _parse_header_values(kind=header.kind, header_text=header_text) - except Exception as error: # noqa: BLE001 - any parse failure is a lint fault - position: tuple[int, int] = source_position(offset=header.start, line_starts=line_starts) - return ( - LintViolation( - file_path=file_path, - line=position[0], - column=position[1], - code=RULE_HEADER_PARSE, - message=f"{header.kind}() header could not be parsed: {error}", - severity=VIOLATION_SEVERITY_FAULT, - engine=LINT_ENGINE_SQLBUILD, - remediation=_header_parse_remediation(kind=header.kind, error=error), - ), - ) - - violations: list[LintViolation] = [] - description: object | None = values.get(_DESCRIPTION_KEY) - span: tuple[int, int] | None = ( - _top_level_description_value_span(header_text=header_text) - if header.kind in DESCRIPTION_HEADER_KINDS - else None - ) - effective_description: object | None = description - if isinstance(description, str) and span is not None: - value_start, value_end = span - effective_description = _decode_quoted_description( - value=header_text[value_start:value_end], - quote=header_text[value_start - 1], - ) - if header.kind in DESCRIPTION_REQUIRED_HEADER_KINDS and ( - not isinstance(effective_description, str) or not effective_description.strip() - ): - violations.append( - _violation_for_header_start( - contents=contents, - file_path=file_path, - header=header, - code=RULE_DESCRIPTION_PRESENT, - message=f"{header.kind}() header requires a description", - remediation=( - f'Add description "" to the {header.kind}() header; ' - "every named resource must be described." - ), - ) - ) - if header.kind in DESCRIPTION_HEADER_KINDS and isinstance(effective_description, str): - formatted_description: str = ( - _wrap_header_description( - description=effective_description, - header_text=header_text, - span=span, - line_width=config.line_width, - ) - if span is not None - else effective_description - ) - if formatted_description.count("\n") + 1 > config.max_description_lines: - violations.append( - _violation_for_header_start( - contents=contents, - file_path=file_path, - header=header, - code=RULE_DESCRIPTION_LENGTH, - message=f"Description exceeds {config.max_description_lines} lines", - remediation=f"Shorten the description to {config.max_description_lines} lines.", - ) - ) - return tuple(violations) - - -def _header_parse_remediation(*, kind: str, error: Exception) -> str: - help_text: str | None = error.help if isinstance(error, DiscoveryError) else None - return f"Correct the {kind}() header syntax." if help_text is None else help_text - - -def _parse_header_values(*, kind: str, header_text: str) -> dict[str, object]: - inner: str = _inner_header_text(header_text=header_text) - return parse_header_values( - header=inner, - file_path=Path(_DESCRIPTION_SENTINEL_PATH), - statement_name=kind, - header_line=None, - ) - - -def external_identifiers_for_headers( - *, contents: str, headers: tuple[HeaderSpan, ...] -) -> tuple[str, ...]: - """Return SQL function argument names that are values rather than columns.""" - for header in headers: - if header.kind != HEADER_KIND_FUNCTION: - continue - values: dict[str, object] = _parse_header_values( - kind=header.kind, - header_text=contents[header.start : header.end], - ) - arguments: object = values.get("arguments") - if isinstance(arguments, dict): - return tuple(sorted(str(name).lower() for name in arguments)) - return () - - -def _lint_header_whitespace( - *, - contents: str, - file_path: Path, - header: HeaderSpan, - line_starts: tuple[int, ...], -) -> tuple[LintViolation, ...]: - header_text: str = contents[header.start : header.end] - if not any(line != line.rstrip() for line in _split_outside_quotes(text=header_text)): - return () - position: tuple[int, int] = source_position(offset=header.start, line_starts=line_starts) - return ( - LintViolation( - file_path=file_path, - line=position[0], - column=position[1], - code=RULE_HEADER_WHITESPACE, - message=f"{header.kind}() header contains trailing whitespace", - severity=VIOLATION_SEVERITY_WARNING, - engine=LINT_ENGINE_SQLBUILD, - remediation="Run sqb format to remove trailing header whitespace.", - ), - ) - - -def _format_header_whitespace(*, contents: str, headers: tuple[HeaderSpan, ...]) -> str: - pieces: list[str] = [] - cursor: int = 0 - header: HeaderSpan - for header in headers: - pieces.append(contents[cursor : header.start]) - pieces.append( - "\n".join( - line.rstrip() - for line in _split_outside_quotes(text=contents[header.start : header.end]) - ) - ) - cursor = header.end - pieces.append(contents[cursor:]) - return "".join(pieces) - - -def _format_description_wrapping( - *, contents: str, headers: tuple[HeaderSpan, ...], config: LintConfig -) -> str: - updated: str = contents - for header in reversed(headers): - if header.kind not in DESCRIPTION_HEADER_KINDS: - continue - header_text: str = updated[header.start : header.end] - span: tuple[int, int] | None = _top_level_description_value_span(header_text=header_text) - if span is None: - continue - try: - description: object | None = _parse_header_values( - kind=header.kind, - header_text=header_text, - ).get(_DESCRIPTION_KEY) - except ModelSqlParseError: - continue - if not isinstance(description, str): - continue - value_start, value_end = span - quote: str = header_text[value_start - 1] - wrapped: str = _wrap_header_description( - description=_decode_quoted_description( - value=header_text[value_start:value_end], quote=quote - ), - header_text=header_text, - span=span, - line_width=config.line_width, - ) - escaped: str = _escape_quoted_value(value=wrapped, quote=quote) - rewritten_header: str = header_text[:value_start] + escaped + header_text[value_end:] - updated = updated[: header.start] + rewritten_header + updated[header.end :] - return updated - - -def _wrap_header_description( - *, description: str, header_text: str, span: tuple[int, int], line_width: int -) -> str: - value_start, value_end = span - line_start: int = header_text.rfind(_NEWLINE, 0, value_start) + 1 - line_end: int = header_text.find(_NEWLINE, value_end) - if line_end < 0: - line_end = len(header_text) - closing_suffix_width: int = len(header_text[value_end:line_end].rstrip()) - wrapped_line_width: int = max(1, line_width - closing_suffix_width) - first_line_width: int = max( - 1, - line_width - (value_start - line_start) - closing_suffix_width, - ) - return _wrap_description( - description=description, - first_line_width=first_line_width, - line_width=wrapped_line_width, - ) - - -def _top_level_description_value_span(*, header_text: str) -> tuple[int, int] | None: - delimiters: list[str] = [] - closing_delimiters: dict[str, str] = {"(": ")", "[": "]", "{": "}"} - index: int = 0 - while index < len(header_text): - character: str = header_text[index] - if character in _QUOTE_CHARACTERS: - index = quoted_value_end(text=header_text, start=index) - continue - if character in closing_delimiters: - delimiters.append(closing_delimiters[character]) - index += 1 - continue - if delimiters and character == delimiters[-1]: - delimiters.pop() - index += 1 - continue - if delimiters == [CLOSING_PAREN_CHARACTER] and ( - character.isalpha() or character == IDENTIFIER_SEPARATOR_CHARACTER - ): - word_end: int = index + 1 - while word_end < len(header_text) and ( - header_text[word_end].isalnum() - or header_text[word_end] == IDENTIFIER_SEPARATOR_CHARACTER - ): - word_end += 1 - if header_text[index:word_end].lower() == _DESCRIPTION_KEY: - quote_start: int = word_end - while quote_start < len(header_text) and header_text[quote_start].isspace(): - quote_start += 1 - if quote_start < len(header_text) and header_text[quote_start] in _QUOTE_CHARACTERS: - quote_end: int = quoted_value_end(text=header_text, start=quote_start) - return quote_start + 1, quote_end - 1 - index = word_end - continue - index += 1 - return None - - -def quoted_value_end(*, text: str, start: int) -> int: - """Return the offset just past the quoted value that opens at start.""" - - quote: str = text[start] - index: int = start + 1 - while index < len(text): - if text[index] == _ESCAPE_CHARACTER and index + 1 < len(text): - index += 2 - continue - if text[index] == quote: - return index + 1 - index += 1 - return len(text) - - -def _wrap_description(*, description: str, first_line_width: int, line_width: int) -> str: - normalized: str = description.replace("\r\n", _NEWLINE).strip() - paragraphs: list[str] = re.split(r"\n\s*\n", normalized) - wrapped_paragraphs: list[str] = [] - for paragraph_index, paragraph in enumerate(paragraphs): - prose: str = " ".join(paragraph.split()) - if not prose: - continue - available_first_width: int = first_line_width if paragraph_index == 0 else line_width - synthetic_indent: str = " " * max(0, line_width - available_first_width) - wrapper: textwrap.TextWrapper = textwrap.TextWrapper( - width=line_width, - initial_indent=synthetic_indent, - subsequent_indent="", - break_long_words=False, - break_on_hyphens=False, - ) - lines: list[str] = wrapper.wrap(prose) - if lines and synthetic_indent: - lines[0] = lines[0][len(synthetic_indent) :] - wrapped_paragraphs.append(_NEWLINE.join(lines)) - return f"{_NEWLINE}{_NEWLINE}".join(wrapped_paragraphs) - - -def _escape_quoted_value(*, value: str, quote: str) -> str: - return value.replace(_ESCAPE_CHARACTER, _ESCAPE_CHARACTER * 2).replace( - quote, _ESCAPE_CHARACTER + quote - ) - - -def _decode_quoted_description(*, value: str, quote: str) -> str: - decoded: list[str] = [] - escape_values: dict[str, str] = { - "b": "\b", - "f": "\f", - "n": _NEWLINE, - "r": "\r", - "t": "\t", - } - index: int = 0 - while index < len(value): - character: str = value[index] - if character != _ESCAPE_CHARACTER or index + 1 >= len(value): - decoded.append(character) - index += 1 - continue - escaped: str = value[index + 1] - if escaped in {_ESCAPE_CHARACTER, quote}: - decoded.append(escaped) - index += 2 - continue - if escaped in escape_values: - decoded.append(escape_values[escaped]) - index += 2 - continue - unicode_width: int = {"u": 4, "U": 8}.get(escaped, 0) - digits: str = value[index + 2 : index + 2 + unicode_width] - if ( - unicode_width - and len(digits) == unicode_width - and all(character in _HEXADECIMAL_CHARACTERS for character in digits) - ): - try: - codepoint: int = int(digits, 16) - except ValueError: - pass - else: - if codepoint <= _UNICODE_MAX_CODEPOINT and not ( - _UNICODE_SURROGATE_START <= codepoint <= _UNICODE_SURROGATE_END - ): - decoded.append(chr(codepoint)) - index += unicode_width + 2 - continue - decoded.extend((_ESCAPE_CHARACTER, escaped)) - index += 2 - return "".join(decoded) - - -def _relocate_leading_comment( - *, contents: str, file_path: Path, config: LintConfig, headers: tuple[HeaderSpan, ...] -) -> _RelocationOutcome: - if not headers or headers[0].kind != HEADER_KIND_MODEL: - return _RelocationOutcome(contents=contents, faults=()) - header: HeaderSpan = headers[0] - comment_text: str | None = _leading_comment_text(preamble=contents[: header.start]) - if comment_text is None: - return _RelocationOutcome(contents=contents, faults=()) - - header_text: str = contents[header.start : header.end] - try: - values: dict[str, object] = _parse_header_values(kind=header.kind, header_text=header_text) - except Exception: # noqa: BLE001 - leave unparseable headers to the parse rule - return _RelocationOutcome(contents=contents, faults=()) - if isinstance(values.get(_DESCRIPTION_KEY), str): - return _RelocationOutcome(contents=contents, faults=()) - - description_prefix_width: int = len(_HEADER_INDENT) + len(_DESCRIPTION_KEY) + 2 - closing_description_syntax_width: int = len('",') - relocated_description: str = _wrap_description( - description=comment_text, - first_line_width=max( - 1, - config.line_width - description_prefix_width - closing_description_syntax_width, - ), - line_width=max(1, config.line_width - closing_description_syntax_width), - ) - faults: list[LintViolation] = [] - if relocated_description.count("\n") + 1 > config.max_description_lines: - position: tuple[int, int] = source_position( - offset=header.start, line_starts=source_line_starts(contents) - ) - faults.append( - LintViolation( - file_path=file_path, - line=position[0], - column=position[1], - code=RULE_LEADING_COMMENT_DESCRIPTION, - message=(f"Relocated leading comment exceeds {config.max_description_lines} lines"), - severity=VIOLATION_SEVERITY_FAULT, - engine=LINT_ENGINE_SQLBUILD, - remediation=( - f"Shorten the relocated description to {config.max_description_lines} lines." - ), - ) - ) - - escaped_description: str = _escape_description(relocated_description) - insertion: str = f'\n{_HEADER_INDENT}{_DESCRIPTION_KEY} "{escaped_description}",' - updated_header: str = _insert_after_open_paren(header_text=header_text, insertion=insertion) - return _RelocationOutcome( - contents=updated_header + contents[header.end :], faults=tuple(faults) - ) - - -def _leading_comment_text(*, preamble: str) -> str | None: - stripped: str = preamble.strip() - if not stripped: - return None - if stripped.startswith(_BLOCK_COMMENT_START): - end_index: int = stripped.find(_BLOCK_COMMENT_END, len(_BLOCK_COMMENT_START)) - if end_index < 0: - return None - return stripped[len(_BLOCK_COMMENT_START) : end_index] - lines: list[str] = [] - collecting: bool = False - raw_line: str - for raw_line in preamble.splitlines(): - candidate: str = raw_line.strip() - if candidate.startswith(_LINE_COMMENT_PREFIX): - lines.append(candidate[len(_LINE_COMMENT_PREFIX) :].lstrip()) - collecting = True - continue - if collecting: - break - if candidate: - return None - if not lines or not collecting: - return None - return "\n".join(lines) - - -def _escape_description(description: str) -> str: - return description.replace("\\", "\\\\").replace('"', '\\"').replace("\r\n", "\n") - - -def _insert_after_open_paren(*, header_text: str, insertion: str) -> str: - open_index: int = header_text.find("(") - if open_index < 0: - return header_text - return header_text[: open_index + 1] + insertion + header_text[open_index + 1 :] - - -def _split_outside_quotes(*, text: str) -> list[str]: - lines: list[str] = [] - current: list[str] = [] - in_quote: str | None = None - index: int = 0 - length: int = len(text) - while index < length: - character: str = text[index] - if in_quote is not None: - current.append(character) - if character == _ESCAPE_CHARACTER and index + 1 < length: - current.append(text[index + 1]) - index += 2 - continue - if character == in_quote: - in_quote = None - index += 1 - continue - if character in _QUOTE_CHARACTERS: - in_quote = character - current.append(character) - index += 1 - continue - if character == _NEWLINE: - lines.append("".join(current)) - current = [] - index += 1 - continue - current.append(character) - index += 1 - lines.append("".join(current)) - return lines - - -def _violation_for_header_start( - *, - contents: str, - file_path: Path, - header: HeaderSpan, - code: str, - message: str, - remediation: str, -) -> LintViolation: - position: tuple[int, int] = source_position( - offset=header.start, line_starts=source_line_starts(contents) - ) - return LintViolation( - file_path=file_path, - line=position[0], - column=position[1], - code=code, - message=message, - severity=VIOLATION_SEVERITY_FAULT, - engine=LINT_ENGINE_SQLBUILD, - remediation=remediation, - ) - - -def _inner_header_text(*, header_text: str) -> str: - open_index: int = header_text.find("(") - close_index: int = header_text.rfind(")") - if open_index < 0 or close_index <= open_index: - return "" - return header_text[open_index + 1 : close_index] - - -def _wrap_header_audits(*, contents: str, headers: tuple[HeaderSpan, ...], line_width: int) -> str: - """Split audit lists in header lines over `line_width`; keep only value-preserving rewrites.""" - - updated: str = contents - for header in reversed(headers): - header_text: str = updated[header.start : header.end] - wrapped: str = _wrap_header(header_text=header_text, line_width=line_width) - if wrapped == header_text: - continue - if _header_values_or_unique(kind=header.kind, header_text=wrapped) != ( - _header_values_or_unique(kind=header.kind, header_text=header_text) - ): - continue - updated = updated[: header.start] + wrapped + updated[header.end :] - return updated - - -def _wrap_header(*, header_text: str, line_width: int) -> str: - text: str = header_text - for _ in range(_AUDIT_WRAP_MAX_PASSES): - exploded: str | None = _explode_first_long_line(text=text, line_width=line_width) - if exploded is None: - return text - text = exploded - return text - - -def _explode_first_long_line(*, text: str, line_width: int) -> str | None: - groups: tuple[_Group, ...] = _audit_groups(text=text) - line_start: int = 0 - for line in text.split("\n"): - line_end: int = line_start + len(line) - if len(line) > line_width and _LINE_COMMENT_PREFIX not in line: - candidates: list[_Group] = [ - group - for group in groups - if line_start <= group.open_index - and group.close_index < line_end - and group.close_index - line_start >= line_width - and text[group.open_index + 1 : group.close_index].strip() - ] - if candidates: - chosen: _Group = min(candidates, key=lambda group: group.depth) - indent: str = line[: len(line) - len(line.lstrip())] - return _explode(text=text, group=chosen, indent=indent) - line_start = line_end + 1 - return None - - -def _explode(*, text: str, group: _Group, indent: str) -> str: - bounds: list[int] = [group.open_index, *group.commas, group.close_index] - items: list[str] = [ - text[start + 1 : end].strip() for start, end in zip(bounds[:-1], bounds[1:], strict=True) - ] - items = [item for item in items if item] - inner: str = indent + _HEADER_INDENT - body: str = "".join(f"\n{inner}{item}," for item in items) - return f"{text[: group.open_index + 1]}{body}\n{indent}{text[group.close_index :]}" - - -def _audit_groups(*, text: str) -> tuple[_Group, ...]: - """Bracket groups at or inside each `audits [...]` value, outside quoted text.""" - - groups: list[_Group] = [] - open_stack: list[tuple[int, str, list[int]]] = [] - audit_depths: list[int] = [] - index: int = 0 - while index < len(text): - character: str = text[index] - if character in _QUOTE_CHARACTERS: - index = quoted_value_end(text=text, start=index) - continue - if character in _OPENERS: - if character == _AUDIT_LIST_OPENER and _AUDITS_KEY.match( - text, _audits_key_start(text=text, bracket=index) - ): - audit_depths.append(len(open_stack)) - open_stack.append((index, _OPENERS[character], [])) - elif open_stack and character == open_stack[-1][1]: - start, _, commas = open_stack.pop() - if audit_depths and len(open_stack) >= audit_depths[-1]: - groups.append( - _Group( - open_index=start, - close_index=index, - depth=len(open_stack), - commas=tuple(commas), - ) - ) - if audit_depths and len(open_stack) == audit_depths[-1]: - audit_depths.pop() - elif open_stack and character == _HEADER_ITEM_SEPARATOR: - open_stack[-1][2].append(index) - index += 1 - return tuple(groups) - - -def _audits_key_start(*, text: str, bracket: int) -> int: - start: int = bracket - while start > 0 and text[start - 1].isspace(): - start -= 1 - return max(0, start - len(_AUDITS_KEYWORD)) - - -def drop_resolved_description_faults( - *, violations: list[LintViolation], project_dir: Path -) -> list[LintViolation]: - """Drop `description-present` faults for models described outside their header.""" - - if not any(violation.code == RULE_DESCRIPTION_PRESENT for violation in violations): - return violations - described: frozenset[Path] = frozenset( - path.resolve() - for path in ModelDescriptionResolution.externally_described_models(project_dir=project_dir) - ) - return [ - violation - for violation in violations - if violation.code != RULE_DESCRIPTION_PRESENT - or violation.file_path.resolve() not in described - ] - - -def source_line_starts(contents: str) -> tuple[int, ...]: - """Return the offset where each line of contents starts.""" - - starts: list[int] = [0] - index: int = contents.find("\n") - while index != -1: - starts.append(index + 1) - index = contents.find("\n", index + 1) - return tuple(starts) - - -def source_position(*, offset: int, line_starts: tuple[int, ...]) -> tuple[int, int]: - """Map a zero-based offset to a one-based line and column.""" - - line_index: int = bisect_right(line_starts, offset) - 1 - return line_index + 1, offset - line_starts[line_index] + 1 diff --git a/src/sqlbuild/lint/_helpers/native_format.py b/src/sqlbuild/lint/_helpers/native_format.py index 48911aa3e8..a8b5af6bf0 100644 --- a/src/sqlbuild/lint/_helpers/native_format.py +++ b/src/sqlbuild/lint/_helpers/native_format.py @@ -9,13 +9,14 @@ from typing import Any import sqlbuild._native as _native +from sqlbuild._native import scan_headers from sqlbuild.compiler.compile.constants import OMITTED_CEREMONIAL_SELECT_SQL from sqlbuild.compiler.discovery.main.omitted_ceremonial_select import ( omitted_ceremonial_select_offset, ) from sqlbuild.compiler.sql_analysis.constants import POLYGLOT_MAX_FUNCTION_CALL_DEPTH from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.lint._helpers.headers import lint_body_ranges, scan_headers +from sqlbuild.lint._helpers.headers import lint_body_ranges from sqlbuild.lint._helpers.sqlbuild_tokens import neutralize_interpolation, restore_interpolation from sqlbuild.lint.constants import ( CARRIAGE_RETURN_LINE_FEED, diff --git a/src/sqlbuild/lint/_helpers/native_sql.py b/src/sqlbuild/lint/_helpers/native_sql.py index 9b3f5fad82..631a5c1506 100644 --- a/src/sqlbuild/lint/_helpers/native_sql.py +++ b/src/sqlbuild/lint/_helpers/native_sql.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +from bisect import bisect_right from pathlib import Path from threading import Event from typing import Any, cast @@ -32,7 +33,6 @@ omitted_ceremonial_select_offset, ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.lint._helpers.native import source_line_starts, source_position from sqlbuild.lint._helpers.sqlbuild_tokens import interpolation_text_at from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.constants import ( @@ -501,3 +501,21 @@ def _violation_message( if token is None: return f"{message} {GENERATED_SQL_MESSAGE_SUFFIX}" return f"{message} {GENERATED_SQL_MESSAGE_TEMPLATE.format(token=token)}" + + +def source_line_starts(contents: str) -> tuple[int, ...]: + """Return the offset where each line of contents starts.""" + + starts: list[int] = [0] + index: int = contents.find("\n") + while index != -1: + starts.append(index + 1) + index = contents.find("\n", index + 1) + return tuple(starts) + + +def source_position(*, offset: int, line_starts: tuple[int, ...]) -> tuple[int, int]: + """Map a zero-based offset to a one-based line and column.""" + + line_index: int = bisect_right(line_starts, offset) - 1 + return line_index + 1, offset - line_starts[line_index] + 1 diff --git a/src/sqlbuild/lint/main/run_format.py b/src/sqlbuild/lint/main/run_format.py index 4d86565a2a..83674b12b5 100644 --- a/src/sqlbuild/lint/main/run_format.py +++ b/src/sqlbuild/lint/main/run_format.py @@ -4,20 +4,20 @@ from pathlib import Path -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.classes.fixture_null_autofix import FixtureNullAutofix -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.lint._helpers.fixes import finalize_fix_reports, persist_changes, plan_rule_fixes -from sqlbuild.lint._helpers.headers import scan_headers -from sqlbuild.lint._helpers.native import ( - drop_resolved_description_faults, +from sqlbuild._native import ( format_native_headers, lint_native_headers, prepare_native_header_cache, reject_unparseable_header_rewrites, safe_format_files, + scan_headers, ) +from sqlbuild.compiler.compile.types import TypedSqlValueRenderer +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.planner.classes.fixture_null_autofix import FixtureNullAutofix +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.lint._helpers.fixes import finalize_fix_reports, persist_changes, plan_rule_fixes +from sqlbuild.lint._helpers.headers import drop_resolved_description_faults from sqlbuild.lint._helpers.native_format import format_native_sql_bodies from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations from sqlbuild.lint._helpers.suppressions import apply_suppressions diff --git a/src/sqlbuild/lint/main/run_lint.py b/src/sqlbuild/lint/main/run_lint.py index 5a980006fe..650dc1cd29 100644 --- a/src/sqlbuild/lint/main/run_lint.py +++ b/src/sqlbuild/lint/main/run_lint.py @@ -4,6 +4,7 @@ from pathlib import Path +from sqlbuild._native import lint_native_headers, scan_headers from sqlbuild.compiler.compile.constants import MODEL_DIRECTORY_NAME from sqlbuild.compiler.compile.models import CompiledSqlExpansion, SqlExpansionContext from sqlbuild.compiler.compile.types import TypedSqlValueRenderer @@ -18,8 +19,6 @@ expand_file_bodies, prepare_expanded_files, ) -from sqlbuild.lint._helpers.headers import scan_headers -from sqlbuild.lint._helpers.native import lint_native_headers from sqlbuild.lint._helpers.native_sql import run_native_sql_lint from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations from sqlbuild.lint._helpers.suppressions import apply_suppressions diff --git a/src/sqlbuild/microbatches/_helpers/projection.py b/src/sqlbuild/microbatches/_helpers/projection.py index f1205b3873..26b2a14173 100644 --- a/src/sqlbuild/microbatches/_helpers/projection.py +++ b/src/sqlbuild/microbatches/_helpers/projection.py @@ -5,11 +5,11 @@ import heapq from datetime import datetime -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.cursor_sort_key import cursor_sort_key -from sqlbuild.cursor_algebra.main.max_bound import max_bound -from sqlbuild.cursor_algebra.main.min_bound import min_bound -from sqlbuild.cursor_algebra.main.parse import parse +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_max_bound as max_bound +from sqlbuild._native import cursor_min_bound as min_bound +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_sort_key from sqlbuild.microbatches.models import ( MicrobatchCoverageProjection, MicrobatchEvent, diff --git a/src/sqlbuild/microbatches/main/merge_intervals.py b/src/sqlbuild/microbatches/main/merge_intervals.py index a596cef74d..2040611c91 100644 --- a/src/sqlbuild/microbatches/main/merge_intervals.py +++ b/src/sqlbuild/microbatches/main/merge_intervals.py @@ -1,11 +1,11 @@ """Merge canonical microbatch intervals.""" +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_merge as merge +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render +from sqlbuild._native import cursor_sort_key from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.cursor_sort_key import cursor_sort_key -from sqlbuild.cursor_algebra.main.merge import merge -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.models import AlignedInterval, IntegerValue from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.microbatches.exceptions import MicrobatchStateError diff --git a/src/sqlbuild/playground_templates/dagster/README.md b/src/sqlbuild/playground_templates/dagster/README.md new file mode 100644 index 0000000000..4f3a3a2d0a --- /dev/null +++ b/src/sqlbuild/playground_templates/dagster/README.md @@ -0,0 +1,18 @@ +# SQLBuild + Dagster playground + +This directory is a Dagster code location. `definitions.py` orchestrates the SQLBuild waffle +shop project in `waffle_shop/`, which stays a plain SQLBuild project with no Dagster code inside. + +## Run Dagster + +Install SQLBuild with the Dagster extra, then start Dagster from this directory: + +```bash +uv add "sqlbuild[dagster]" +DAGSTER_IS_DEV_CLI=1 uv run dagster dev -f definitions.py +``` + +Open the Dagster UI, materialize the `waffle_shop_assets` asset group, then run the scenario checks. + +The definitions use `SqlBuildProject.prepare_if_dev()` to generate +`waffle_shop/target/sqlbuild_dag.json` when Dagster starts in dev mode. diff --git a/src/sqlbuild/playground_templates/dagster/definitions.py.txt b/src/sqlbuild/playground_templates/dagster/definitions.py.txt new file mode 100644 index 0000000000..f289d6ea13 --- /dev/null +++ b/src/sqlbuild/playground_templates/dagster/definitions.py.txt @@ -0,0 +1,41 @@ +"""Dagster definitions for the SQLBuild waffle shop playground.""" + +from pathlib import Path + +import dagster as dg + +from sqlbuild.integrations.dagster import ( + SqlBuildCliResource, + SqlBuildProject, + sqlbuild_assets, + sqlbuild_scenario_checks, +) + +PROJECT_DIR = Path(__file__).resolve().parent / "waffle_shop" +SQLBUILD_PROJECT = SqlBuildProject(project_dir=PROJECT_DIR) + +SQLBUILD_PROJECT.prepare_if_dev() + + +@sqlbuild_assets( + project=SQLBUILD_PROJECT, + required_resource_keys={"sqb"}, + include_scenario_checks=False, +) +def waffle_shop_assets(context: dg.AssetExecutionContext): + yield from context.resources.sqb.cli(["build"], context=context).stream() + + +@sqlbuild_scenario_checks( + project=SQLBUILD_PROJECT, + required_resource_keys={"sqb"}, +) +def waffle_shop_scenarios(context: dg.AssetCheckExecutionContext): + yield from context.resources.sqb.cli(["scenario", "test"], context=context).stream() + + +defs = dg.Definitions( + assets=[waffle_shop_assets], + asset_checks=[waffle_shop_scenarios], + resources={"sqb": SqlBuildCliResource(project_dir=SQLBUILD_PROJECT)}, +) diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/README.md b/src/sqlbuild/playground_templates/loader_waffle_shop/README.md similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/README.md rename to src/sqlbuild/playground_templates/loader_waffle_shop/README.md diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/hooks/python/.gitkeep b/src/sqlbuild/playground_templates/loader_waffle_shop/hooks/python/.gitkeep similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/hooks/python/.gitkeep rename to src/sqlbuild/playground_templates/loader_waffle_shop/hooks/python/.gitkeep diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/hooks/sql/.gitkeep b/src/sqlbuild/playground_templates/loader_waffle_shop/hooks/sql/.gitkeep similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/hooks/sql/.gitkeep rename to src/sqlbuild/playground_templates/loader_waffle_shop/hooks/sql/.gitkeep diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/customer_revenue.sql b/src/sqlbuild/playground_templates/loader_waffle_shop/models/customer_revenue.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/customer_revenue.sql rename to src/sqlbuild/playground_templates/loader_waffle_shop/models/customer_revenue.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/fact_waffle_orders.sql b/src/sqlbuild/playground_templates/loader_waffle_shop/models/fact_waffle_orders.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/fact_waffle_orders.sql rename to src/sqlbuild/playground_templates/loader_waffle_shop/models/fact_waffle_orders.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt b/src/sqlbuild/playground_templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt rename to src/sqlbuild/playground_templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml b/src/sqlbuild/playground_templates/loader_waffle_shop/sources/raw.yml similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml rename to src/sqlbuild/playground_templates/loader_waffle_shop/sources/raw.yml diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sqlbuild_project.toml b/src/sqlbuild/playground_templates/loader_waffle_shop/sqlbuild_project.toml similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sqlbuild_project.toml rename to src/sqlbuild/playground_templates/loader_waffle_shop/sqlbuild_project.toml diff --git a/src/sqlbuild/playground_templates/python_nodes/README.md b/src/sqlbuild/playground_templates/python_nodes/README.md new file mode 100644 index 0000000000..afa1154e05 --- /dev/null +++ b/src/sqlbuild/playground_templates/python_nodes/README.md @@ -0,0 +1,14 @@ +# SQLBuild Python Nodes Playground + +This project demonstrates first-class Python nodes in a small local DuckDB project. + +Try: + +```bash +sqb plan --select +fact_orders --select +orders_export +sqb build --select +fact_orders --select +orders_export +sqb check --select check_orders_export +``` + +The project includes a task feeding a loader, a model feeding Python assets through +`ctx.relation(model(...))`, soft skip fan-in, `materialized=False`, and a Python check. diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/hooks/python/.gitkeep b/src/sqlbuild/playground_templates/python_nodes/hooks/python/.gitkeep similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/hooks/python/.gitkeep rename to src/sqlbuild/playground_templates/python_nodes/hooks/python/.gitkeep diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/hooks/sql/.gitkeep b/src/sqlbuild/playground_templates/python_nodes/hooks/sql/.gitkeep similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/hooks/sql/.gitkeep rename to src/sqlbuild/playground_templates/python_nodes/hooks/sql/.gitkeep diff --git a/src/sqlbuild/playground_templates/python_nodes/models/fact_orders.sql b/src/sqlbuild/playground_templates/python_nodes/models/fact_orders.sql new file mode 100644 index 0000000000..7183677fb6 --- /dev/null +++ b/src/sqlbuild/playground_templates/python_nodes/models/fact_orders.sql @@ -0,0 +1,15 @@ +MODEL ( + description "One row per order with its customer and amount", + materialized table, + columns ( + order_id (nullable false, audits [not_null, unique]), + customer_id (nullable false, audits [not_null]), + amount_cents (nullable false, audits [not_null]), + ), +); + +SELECT + order_id, + customer_id, + amount_cents +FROM __source("raw_orders") diff --git a/src/sqlbuild/playground_templates/python_nodes/python/assets/orders_export.py.txt b/src/sqlbuild/playground_templates/python_nodes/python/assets/orders_export.py.txt new file mode 100644 index 0000000000..172b71b1b7 --- /dev/null +++ b/src/sqlbuild/playground_templates/python_nodes/python/assets/orders_export.py.txt @@ -0,0 +1,26 @@ +"""Asset examples for the Python nodes playground.""" + +from sqlbuild.assets import asset +from sqlbuild.refs import model + +from python.tasks.orders import export_window, optional_partner_feed + + +@asset( + depends_on=(model("fact_orders"), optional_partner_feed, export_window), + tags=["python", "external"], + group="exports", + columns=[{"name": "order_count", "type": "INTEGER"}], + column_lineage={"order_count": [{"node": "fact_orders", "column": "order_id"}]}, +) +def orders_export(ctx): + """Read a SQL model relation and describe an external export artifact.""" + + relation = ctx.relation(model("fact_orders")) + rows = ctx.query(f"SELECT COUNT(*) AS order_count FROM {relation}").fetchall() + order_count = int(rows[0][0]) + return ctx.result( + payload={"order_count": order_count}, + metadata={"target_uri": "s3://example-bucket/orders.json", "window": "daily"}, + materialized=False, + ) diff --git a/src/sqlbuild/playground_templates/python_nodes/python/checks/orders_export.py.txt b/src/sqlbuild/playground_templates/python_nodes/python/checks/orders_export.py.txt new file mode 100644 index 0000000000..cc4cd581d9 --- /dev/null +++ b/src/sqlbuild/playground_templates/python_nodes/python/checks/orders_export.py.txt @@ -0,0 +1,22 @@ +"""Check examples for the Python nodes playground.""" + +from sqlbuild.checks import check + +from python.assets.orders_export import orders_export + + +@check(depends_on=orders_export, tags=["quality"], group="exports") +def check_orders_export(ctx): + """Validate the Python asset's same-run metadata and payload.""" + + result = ctx.result_of(node_function=orders_export) + payload = result.payload + metadata = result.metadata + if payload["order_count"] <= 0: + return ctx.fail(message="orders export is empty") + if "target_uri" not in metadata: + return ctx.warn(message="orders export target URI is missing") + return ctx.pass_( + message="orders export is ready", + metadata={"order_count": payload["order_count"]}, + ) diff --git a/src/sqlbuild/playground_templates/python_nodes/python/loaders/orders.py.txt b/src/sqlbuild/playground_templates/python_nodes/python/loaders/orders.py.txt new file mode 100644 index 0000000000..00d91baa37 --- /dev/null +++ b/src/sqlbuild/playground_templates/python_nodes/python/loaders/orders.py.txt @@ -0,0 +1,16 @@ +"""Loader examples for the Python nodes playground.""" + +from sqlbuild.loaders import loader +from python.tasks.orders import prepare_raw_orders + + +@loader(depends_on=(prepare_raw_orders,)) +def raw_orders(ctx): + """Load source rows after the preparation task has completed.""" + + # Loader contexts intentionally keep the SQL ingress API narrow; the dependency makes + # the task part of the same DAG even though this example returns static rows. + return [ + {"order_id": 1, "customer_id": 10, "amount_cents": 1200, "load_batch": 1}, + {"order_id": 2, "customer_id": 20, "amount_cents": 800, "load_batch": 1}, + ] diff --git a/src/sqlbuild/playground_templates/python_nodes/python/tasks/orders.py.txt b/src/sqlbuild/playground_templates/python_nodes/python/tasks/orders.py.txt new file mode 100644 index 0000000000..04367ff839 --- /dev/null +++ b/src/sqlbuild/playground_templates/python_nodes/python/tasks/orders.py.txt @@ -0,0 +1,28 @@ +"""Task examples for the Python nodes playground.""" + +from sqlbuild.tasks import SkipMode, task + + +@task(tags=["python", "ingress"], group="ingress") +def prepare_raw_orders(ctx): + """Prepare a tiny in-memory order extract before the loader runs.""" + + rows = [ + {"order_id": 1, "customer_id": 10, "amount_cents": 1200, "load_batch": 1}, + {"order_id": 2, "customer_id": 20, "amount_cents": 800, "load_batch": 1}, + ] + return ctx.result(payload={"rows": rows}, metadata={"row_count": len(rows)}) + + +@task(tags=["python", "optional"], group="exports") +def optional_partner_feed(ctx): + """Soft-skip an optional upstream without blocking sibling fan-in.""" + + return ctx.skip(reason="partner feed is not configured", mode=SkipMode.SOFT) + + +@task(tags=["python", "optional"], group="exports") +def export_window(ctx): + """Successful sibling that lets downstream fan-in continue.""" + + return ctx.result(payload={"window": "daily"}) diff --git a/src/sqlbuild/playground_templates/python_nodes/sources/raw.yml b/src/sqlbuild/playground_templates/python_nodes/sources/raw.yml new file mode 100644 index 0000000000..e09cff6df6 --- /dev/null +++ b/src/sqlbuild/playground_templates/python_nodes/sources/raw.yml @@ -0,0 +1,14 @@ +sources: + - name: raw_orders + description: Raw orders loaded by the raw_orders Python loader + managed: true + write_strategy: table + columns: + - name: order_id + type: INTEGER + - name: customer_id + type: INTEGER + - name: amount_cents + type: INTEGER + - name: load_batch + type: INTEGER diff --git a/src/sqlbuild/playground_templates/python_nodes/sqlbuild_project.toml b/src/sqlbuild/playground_templates/python_nodes/sqlbuild_project.toml new file mode 100644 index 0000000000..993359304d --- /dev/null +++ b/src/sqlbuild/playground_templates/python_nodes/sqlbuild_project.toml @@ -0,0 +1,16 @@ +name = "python_nodes_demo" +adapter = "duckdb" +default_target = "dev" + +[connections.developer] +database = "python_nodes_demo.duckdb" + +[settings] +default_audit_severity = "warn" + +[defaults] +materialized = "table" + +[targets.dev] +connection = "developer" +schema = "main" diff --git a/src/sqlbuild/playground_templates/rivers/README.md b/src/sqlbuild/playground_templates/rivers/README.md new file mode 100644 index 0000000000..3b9d9df42c --- /dev/null +++ b/src/sqlbuild/playground_templates/rivers/README.md @@ -0,0 +1,24 @@ +# SQLBuild + Rivers playground + +This directory is a Rivers code repository. `definitions.py` orchestrates the SQLBuild waffle +shop project in `waffle_shop/`, which stays a plain SQLBuild project with no Rivers code inside. + +## Run Rivers + +Install SQLBuild with the Rivers extra, then start Rivers from this directory: + +```bash +uv add "sqlbuild[rivers]" +uv run rivers dev definitions +``` + +You can also run a local materialization directly: + +```bash +uv run python definitions.py +``` + +The repository defines a `waffle_shop` job for UI-triggered materialization. + +The definitions use `SqlBuildProject.prepare_if_dev()` to generate +`waffle_shop/target/sqlbuild_dag.json` when `rivers dev` starts with `RIVERS_DEPLOYMENT=dev`. diff --git a/src/sqlbuild/playground_templates/rivers/definitions.py.txt b/src/sqlbuild/playground_templates/rivers/definitions.py.txt new file mode 100644 index 0000000000..117d77da2a --- /dev/null +++ b/src/sqlbuild/playground_templates/rivers/definitions.py.txt @@ -0,0 +1,53 @@ +"""Rivers definitions for the SQLBuild waffle shop playground.""" + +from __future__ import annotations + +import subprocess +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import rivers as rs + +from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets + +PROJECT_DIR = Path(__file__).resolve().parent / "waffle_shop" +SQLBUILD_PROJECT = SqlBuildProject(project_dir=PROJECT_DIR) + +if __name__ == "__main__": + SQLBUILD_PROJECT.prepare() +else: + SQLBUILD_PROJECT.prepare_if_dev() + + +@sqlbuild_assets(project=SQLBUILD_PROJECT) +def waffle_shop_assets(context: Any) -> Iterator[Any]: + completed = subprocess.run( + ["sqb", "build"], + cwd=PROJECT_DIR, + capture_output=True, + check=False, + text=True, + ) + if completed.returncode != 0: + raise RuntimeError(completed.stderr or completed.stdout) + for output_name in context.output_selection: + yield rs.Materialization(output_name=output_name) + + +repo = rs.CodeRepository( + assets=[waffle_shop_assets], + jobs=[ + rs.Job( + name="waffle_shop", + assets=[waffle_shop_assets], + executor=rs.Executor.in_process(), + ), + ], + default_executor=rs.Executor.in_process(), +) + + +if __name__ == "__main__": + result = repo.get_job("waffle_shop").execute() + raise SystemExit(0 if result.success else 1) diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/README.md b/src/sqlbuild/playground_templates/waffle_shop/README.md similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/README.md rename to src/sqlbuild/playground_templates/waffle_shop/README.md diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/table_fn__customer_orders.sql b/src/sqlbuild/playground_templates/waffle_shop/functions/sql/table_fn__customer_orders.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/table_fn__customer_orders.sql rename to src/sqlbuild/playground_templates/waffle_shop/functions/sql/table_fn__customer_orders.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/udf__is_completed_order.sql b/src/sqlbuild/playground_templates/waffle_shop/functions/sql/udf__is_completed_order.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/udf__is_completed_order.sql rename to src/sqlbuild/playground_templates/waffle_shop/functions/sql/udf__is_completed_order.sql diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/__init__.py b/src/sqlbuild/playground_templates/waffle_shop/hooks/python/.gitkeep similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/janitor_output/__init__.py rename to src/sqlbuild/playground_templates/waffle_shop/hooks/python/.gitkeep diff --git a/src/sqlbuild/cursor_algebra/main/__init__.py b/src/sqlbuild/playground_templates/waffle_shop/hooks/sql/.gitkeep similarity index 100% rename from src/sqlbuild/cursor_algebra/main/__init__.py rename to src/sqlbuild/playground_templates/waffle_shop/hooks/sql/.gitkeep diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/macros/currency.py.txt b/src/sqlbuild/playground_templates/waffle_shop/macros/currency.py.txt similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/macros/currency.py.txt rename to src/sqlbuild/playground_templates/waffle_shop/macros/currency.py.txt diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/materializations/partition_tracked.py.txt b/src/sqlbuild/playground_templates/waffle_shop/materializations/partition_tracked.py.txt similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/materializations/partition_tracked.py.txt rename to src/sqlbuild/playground_templates/waffle_shop/materializations/partition_tracked.py.txt diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/intermediate/customer_status_snapshot.sql b/src/sqlbuild/playground_templates/waffle_shop/models/intermediate/customer_status_snapshot.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/intermediate/customer_status_snapshot.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/intermediate/customer_status_snapshot.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/intermediate/order_status_index.sql b/src/sqlbuild/playground_templates/waffle_shop/models/intermediate/order_status_index.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/intermediate/order_status_index.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/intermediate/order_status_index.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/_sqlbuild/_audits/generic/expression_is_true.sql b/src/sqlbuild/playground_templates/waffle_shop/models/marts/_sqlbuild/_audits/generic/expression_is_true.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/_sqlbuild/_audits/generic/expression_is_true.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/marts/_sqlbuild/_audits/generic/expression_is_true.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/_sqlbuild/_macros/currency.py.txt b/src/sqlbuild/playground_templates/waffle_shop/models/marts/_sqlbuild/_macros/currency.py.txt similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/_sqlbuild/_macros/currency.py.txt rename to src/sqlbuild/playground_templates/waffle_shop/models/marts/_sqlbuild/_macros/currency.py.txt diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/_sqlbuild/_macros/datetime.py.txt b/src/sqlbuild/playground_templates/waffle_shop/models/marts/_sqlbuild/_macros/datetime.py.txt similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/_sqlbuild/_macros/datetime.py.txt rename to src/sqlbuild/playground_templates/waffle_shop/models/marts/_sqlbuild/_macros/datetime.py.txt diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/daily_activity_rollup.sql b/src/sqlbuild/playground_templates/waffle_shop/models/marts/daily_activity_rollup.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/daily_activity_rollup.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/marts/daily_activity_rollup.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/daily_order_partitioned.sql b/src/sqlbuild/playground_templates/waffle_shop/models/marts/daily_order_partitioned.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/daily_order_partitioned.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/marts/daily_order_partitioned.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/daily_revenue.sql b/src/sqlbuild/playground_templates/waffle_shop/models/marts/daily_revenue.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/daily_revenue.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/marts/daily_revenue.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/dim_customers.sql b/src/sqlbuild/playground_templates/waffle_shop/models/marts/dim_customers.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/dim_customers.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/marts/dim_customers.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/fact_orders.sql b/src/sqlbuild/playground_templates/waffle_shop/models/marts/fact_orders.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/fact_orders.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/marts/fact_orders.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/hourly_activity_with_daily_context.sql b/src/sqlbuild/playground_templates/waffle_shop/models/marts/hourly_activity_with_daily_context.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/hourly_activity_with_daily_context.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/marts/hourly_activity_with_daily_context.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/hourly_order_activity.sql b/src/sqlbuild/playground_templates/waffle_shop/models/marts/hourly_order_activity.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/marts/hourly_order_activity.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/marts/hourly_order_activity.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/staging/stg_customers.sql b/src/sqlbuild/playground_templates/waffle_shop/models/staging/stg_customers.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/staging/stg_customers.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/staging/stg_customers.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/staging/stg_orders.sql b/src/sqlbuild/playground_templates/waffle_shop/models/staging/stg_orders.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/staging/stg_orders.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/staging/stg_orders.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/staging/stg_payments.sql b/src/sqlbuild/playground_templates/waffle_shop/models/staging/stg_payments.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/models/staging/stg_payments.sql rename to src/sqlbuild/playground_templates/waffle_shop/models/staging/stg_payments.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/python/loaders/waffle_sources.py.txt b/src/sqlbuild/playground_templates/waffle_shop/python/loaders/waffle_sources.py.txt similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/python/loaders/waffle_sources.py.txt rename to src/sqlbuild/playground_templates/waffle_shop/python/loaders/waffle_sources.py.txt diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/seeds/lookups.yml b/src/sqlbuild/playground_templates/waffle_shop/seeds/lookups.yml similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/seeds/lookups.yml rename to src/sqlbuild/playground_templates/waffle_shop/seeds/lookups.yml diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/seeds/waffle_types.csv b/src/sqlbuild/playground_templates/waffle_shop/seeds/waffle_types.csv similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/seeds/waffle_types.csv rename to src/sqlbuild/playground_templates/waffle_shop/seeds/waffle_types.csv diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml b/src/sqlbuild/playground_templates/waffle_shop/sources/raw.yml similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml rename to src/sqlbuild/playground_templates/waffle_shop/sources/raw.yml diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sqlbuild_project.toml b/src/sqlbuild/playground_templates/waffle_shop/sqlbuild_project.toml similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sqlbuild_project.toml rename to src/sqlbuild/playground_templates/waffle_shop/sqlbuild_project.toml diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/scenarios/daily_revenue_minimal.sql b/src/sqlbuild/playground_templates/waffle_shop/tests/scenarios/daily_revenue_minimal.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/scenarios/daily_revenue_minimal.sql rename to src/sqlbuild/playground_templates/waffle_shop/tests/scenarios/daily_revenue_minimal.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/scenarios/daily_revenue_multi_order.sql b/src/sqlbuild/playground_templates/waffle_shop/tests/scenarios/daily_revenue_multi_order.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/scenarios/daily_revenue_multi_order.sql rename to src/sqlbuild/playground_templates/waffle_shop/tests/scenarios/daily_revenue_multi_order.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/unit/test_daily_revenue_chain.sql b/src/sqlbuild/playground_templates/waffle_shop/tests/unit/test_daily_revenue_chain.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/unit/test_daily_revenue_chain.sql rename to src/sqlbuild/playground_templates/waffle_shop/tests/unit/test_daily_revenue_chain.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/unit/test_fact_orders.sql b/src/sqlbuild/playground_templates/waffle_shop/tests/unit/test_fact_orders.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/unit/test_fact_orders.sql rename to src/sqlbuild/playground_templates/waffle_shop/tests/unit/test_fact_orders.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/unit/test_line_total_cents_macro.sql b/src/sqlbuild/playground_templates/waffle_shop/tests/unit/test_line_total_cents_macro.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/unit/test_line_total_cents_macro.sql rename to src/sqlbuild/playground_templates/waffle_shop/tests/unit/test_line_total_cents_macro.sql diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/unit/test_stg_orders.sql b/src/sqlbuild/playground_templates/waffle_shop/tests/unit/test_stg_orders.sql similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/tests/unit/test_stg_orders.sql rename to src/sqlbuild/playground_templates/waffle_shop/tests/unit/test_stg_orders.sql diff --git a/src/sqlbuild/presentation/_helpers/alignment.py b/src/sqlbuild/presentation/_helpers/alignment.py deleted file mode 100644 index 42237dabb4..0000000000 --- a/src/sqlbuild/presentation/_helpers/alignment.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Aligned CLI text implementations.""" - -from __future__ import annotations - - -def resolve_name_column_width(*, names: list[str] | tuple[str, ...], min_width: int = 20) -> int: - if not names: - return min_width - return max(min_width, max(len(name) for name in names)) - - -def format_aligned_name_value( - *, - plain_name: str, - styled_name: str, - value: str, - name_column_width: int, - prefix: str = " ", -) -> str: - padding: str = " " * max(0, name_column_width - len(plain_name)) - return f"{prefix}{styled_name}{padding} {value}" diff --git a/src/sqlbuild/presentation/_helpers/display.py b/src/sqlbuild/presentation/_helpers/display.py deleted file mode 100644 index 341d685f1a..0000000000 --- a/src/sqlbuild/presentation/_helpers/display.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Bounded human-output implementations.""" - -from __future__ import annotations - -from collections.abc import Sequence - -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.models import DisplayOptions - - -def visible_entries[T](*, entries: Sequence[T], options: DisplayOptions) -> Sequence[T]: - if options.max_entries_per_section is None: - return entries - return entries[: options.max_entries_per_section] - - -def append_overflow_line( - *, lines: list[str], total_count: int, visible_count: int, indent: str, options: DisplayOptions -) -> list[str]: - hidden_count: int = total_count - visible_count - if hidden_count <= 0: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.append( - style.muted( - f"{indent}... and {hidden_count} more (use {options.overflow_flag} to show all)" - ) - ) - return lines diff --git a/src/sqlbuild/presentation/_helpers/structure.py b/src/sqlbuild/presentation/_helpers/structure.py index 766ce030c1..89f34f3ef6 100644 --- a/src/sqlbuild/presentation/_helpers/structure.py +++ b/src/sqlbuild/presentation/_helpers/structure.py @@ -2,8 +2,6 @@ from __future__ import annotations -from collections.abc import Callable - from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.constants import ( PHASE_FAIL_GLYPH, @@ -11,7 +9,6 @@ TREE_BRANCH_GLYPH, TREE_LAST_GLYPH, ) -from sqlbuild.presentation.types import CompletionState def tree_connector(*, style: CliStyle, last: bool) -> str: @@ -29,29 +26,6 @@ def format_surface_header(*, style: CliStyle, title: str, context: str | None = return rendered -def count_header_style( - *, style: CliStyle, title_style: Callable[[str], str] | None = None -) -> Callable[[str], str]: - """Return a header style that renders ``Title (count)`` as a bold title and dim count.""" - - resolved_title_style: Callable[[str], str] = title_style or style.section - - def render(text: str) -> str: - title, separator, count = text.rpartition(" (") - if not separator or not text.endswith(")"): - return resolved_title_style(text) - return f"{resolved_title_style(title)} {style.muted(f'({count}')}" - - return render - - -def format_count_noun(*, count: int, singular: str, plural: str | None = None) -> str: - """Render a count with the singular noun for one and the plural noun otherwise.""" - - noun: str = singular if count == 1 else (plural or f"{singular}s") - return f"{count} {noun}" - - def format_phase_line(*, style: CliStyle, ok: bool, label: str, summary: str | None = None) -> str: """Render a concise phase-completion line: state glyph, label, dim summary.""" @@ -61,30 +35,3 @@ def format_phase_line(*, style: CliStyle, ok: bool, label: str, summary: str | N if summary: rendered = f"{rendered} {style.muted(summary)}" return rendered - - -def format_status_cell(*, style: CliStyle, status: str, width: int = 6) -> str: - """Render a fixed-width status cell padded on the plain text, not the ANSI text.""" - - padding: str = " " * max(0, width - len(status)) - return f"{style.status(status=status)}{padding}" - - -def format_completion_line( - *, style: CliStyle, state: CompletionState, label: str, summary: str | None = None -) -> str: - """Render a single-line completion summary: state glyph, label, trailing summary.""" - - if state == CompletionState.FAIL: - glyph: str = style.error(PHASE_FAIL_GLYPH) - rendered_label: str = style.error_strong(label) - elif state == CompletionState.WARN: - glyph = style.warning(PHASE_OK_GLYPH) - rendered_label = style.warning_strong(label) - else: - glyph = style.success(PHASE_OK_GLYPH) - rendered_label = label - rendered: str = f"{glyph} {rendered_label}" - if summary: - rendered = f"{rendered} {summary}" - return rendered diff --git a/src/sqlbuild/presentation/_helpers/summary_footer.py b/src/sqlbuild/presentation/_helpers/summary_footer.py deleted file mode 100644 index e3499cc514..0000000000 --- a/src/sqlbuild/presentation/_helpers/summary_footer.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Status summary footer rendering implementation.""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_summary_footer( - *, counts: tuple[tuple[str, int], ...], use_color: bool, elapsed: str | None = None -) -> str: - style: CliStyle = CliStyle(use_color=use_color) - parts: list[str] = [] - label: str - value: int - success_labels: frozenset[str] = frozenset( - {"PASS", "OK", "SYNC_PASS", "REFRESH_PASS", "UNCHANGED"} - ) - warning_labels: frozenset[str] = frozenset({"WARN", "TOLERATED", "UNKNOWN"}) - error_labels: frozenset[str] = frozenset({"FAIL", "ERROR", "SYNC_FAIL", "REFRESH_FAIL"}) - skip_labels: frozenset[str] = frozenset({"SKIP"}) - for label, value in counts: - normalized: str = label.upper() - rendered_value: str = style.value(str(value)) - if normalized in success_labels: - rendered_value = style.success(str(value)) - elif normalized in warning_labels: - rendered_value = style.warning(str(value)) - elif normalized in error_labels: - rendered_value = style.error(str(value)) - elif normalized in skip_labels: - rendered_value = style.muted(str(value)) - parts.append(f"{style.label(f'{label}=')}{rendered_value}") - rendered: str = " ".join(parts) - if elapsed is not None: - rendered = f"{rendered} {style.muted(f'({elapsed})')}" - return rendered diff --git a/src/sqlbuild/presentation/classes/cli_document.py b/src/sqlbuild/presentation/classes/cli_document.py deleted file mode 100644 index 5f670c2445..0000000000 --- a/src/sqlbuild/presentation/classes/cli_document.py +++ /dev/null @@ -1,58 +0,0 @@ -"""CLI document builder class.""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.cli_style import CliStyle - - -class CliDocument: - """Build simple static CLI output with semantic styling.""" - - def __init__(self, style: CliStyle) -> None: - self._style: CliStyle = style - self._lines: list[str] = [] - - def header(self, *, text: str, suffix: str | None = None) -> None: - rendered: str = self._style.title(text) - if suffix is not None: - rendered = f"{rendered} {suffix}" - self._lines.append(rendered) - - def blank(self) -> None: - self._lines.append("") - - def field(self, *, label: str, value: str, value_padding: str = " ") -> None: - self._lines.append(f" {self._style.value(label)}:{value_padding}{value}") - - def fields(self, *, rows: tuple[tuple[str, str], ...], label_width: int | None = None) -> None: - for label, value in rows: - rendered_label: str = f"{label:<{label_width}}" if label_width is not None else label - self._lines.append(f" {self._style.value(rendered_label)} {value}") - - def section(self, text: str) -> None: - self._lines.append(f"{self._style.section(text)}:") - - def title_section(self, text: str) -> None: - self._lines.append(f"{self._style.title(text)}:") - - def line(self, text: str) -> None: - self._lines.append(text) - - def items(self, values: tuple[str, ...]) -> None: - for value in values: - self._lines.append(f" {self._style.object_name(value)}") - - def command_line(self, *, prefix: str, command: str, style_command: bool = True) -> None: - rendered_command: str = self._style.command(command) if style_command else command - self._lines.append(f"{prefix}{rendered_command}") - - def commands( - self, *, commands: tuple[str, ...], numbered: bool = False, style_command: bool = True - ) -> None: - for index, command in enumerate(commands, start=1): - prefix: str = f" {index}. " if numbered else self._style.command(" ") - self.command_line(prefix=prefix, command=command, style_command=style_command) - - def render(self, *, trailing_newline: bool = True) -> str: - rendered: str = "\n".join(self._lines) - return rendered + "\n" if trailing_newline else rendered diff --git a/src/sqlbuild/presentation/main/maybe_status.py b/src/sqlbuild/presentation/main/_maybe_status.py similarity index 100% rename from src/sqlbuild/presentation/main/maybe_status.py rename to src/sqlbuild/presentation/main/_maybe_status.py diff --git a/src/sqlbuild/presentation/main/aligned_name_value.py b/src/sqlbuild/presentation/main/aligned_name_value.py deleted file mode 100644 index 1eb7bd11f6..0000000000 --- a/src/sqlbuild/presentation/main/aligned_name_value.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public aligned name-value formatting entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.alignment import ( - format_aligned_name_value as _format_aligned_name_value, -) - - -def format_aligned_name_value( - *, - plain_name: str, - styled_name: str, - value: str, - name_column_width: int, - prefix: str = " ", -) -> str: - """Format one aligned name/value row using a plain-text width basis.""" - - return _format_aligned_name_value( - plain_name=plain_name, - styled_name=styled_name, - value=value, - name_column_width=name_column_width, - prefix=prefix, - ) diff --git a/src/sqlbuild/presentation/main/append_overflow_line.py b/src/sqlbuild/presentation/main/append_overflow_line.py deleted file mode 100644 index 5c22fa8f56..0000000000 --- a/src/sqlbuild/presentation/main/append_overflow_line.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public bounded-output overflow entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.display import append_overflow_line as _append_overflow_line -from sqlbuild.presentation.models import DisplayOptions - - -def append_overflow_line( - *, lines: list[str], total_count: int, visible_count: int, indent: str, options: DisplayOptions -) -> list[str]: - """Append a standard overflow hint when a section is truncated.""" - - return _append_overflow_line( - lines=lines, - total_count=total_count, - visible_count=visible_count, - indent=indent, - options=options, - ) diff --git a/src/sqlbuild/presentation/main/completion_line.py b/src/sqlbuild/presentation/main/completion_line.py deleted file mode 100644 index 0cccf89113..0000000000 --- a/src/sqlbuild/presentation/main/completion_line.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public completion summary line rendering entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.structure import ( - format_completion_line as _format_completion_line, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.types import CompletionState - - -def format_completion_line( - *, style: CliStyle, state: CompletionState, label: str, summary: str | None = None -) -> str: - """Render a single-line completion summary: state glyph, label, trailing summary.""" - - return _format_completion_line(style=style, state=state, label=label, summary=summary) diff --git a/src/sqlbuild/presentation/main/count_header.py b/src/sqlbuild/presentation/main/count_header.py deleted file mode 100644 index 65078f7993..0000000000 --- a/src/sqlbuild/presentation/main/count_header.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public counted section header style entry.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.presentation._helpers.structure import count_header_style as _count_header_style -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def count_header_style( - *, style: CliStyle, title_style: Callable[[str], str] | None = None -) -> Callable[[str], str]: - """Return a header style that renders ``Title (count)`` as a bold title and dim count.""" - - return _count_header_style(style=style, title_style=title_style) diff --git a/src/sqlbuild/presentation/main/count_noun.py b/src/sqlbuild/presentation/main/count_noun.py deleted file mode 100644 index 2fa3a2f787..0000000000 --- a/src/sqlbuild/presentation/main/count_noun.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public counted noun rendering entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.structure import format_count_noun as _format_count_noun - - -def format_count_noun(*, count: int, singular: str, plural: str | None = None) -> str: - """Render a count with the singular noun for one and the plural noun otherwise.""" - - return _format_count_noun(count=count, singular=singular, plural=plural) diff --git a/src/sqlbuild/presentation/main/resolve_name_column_width.py b/src/sqlbuild/presentation/main/resolve_name_column_width.py deleted file mode 100644 index bfee3f912f..0000000000 --- a/src/sqlbuild/presentation/main/resolve_name_column_width.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public aligned name-column width entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.alignment import ( - resolve_name_column_width as _resolve_name_column_width, -) - - -def resolve_name_column_width(*, names: list[str] | tuple[str, ...], min_width: int = 20) -> int: - """Resolve a left-column width from the longest visible name.""" - - return _resolve_name_column_width(names=names, min_width=min_width) diff --git a/src/sqlbuild/presentation/main/status_cell.py b/src/sqlbuild/presentation/main/status_cell.py deleted file mode 100644 index f749c067b2..0000000000 --- a/src/sqlbuild/presentation/main/status_cell.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public fixed-width status cell rendering entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.structure import format_status_cell as _format_status_cell -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_status_cell(*, style: CliStyle, status: str, width: int = 6) -> str: - """Render a fixed-width status cell padded on the plain text, not the ANSI text.""" - - return _format_status_cell(style=style, status=status, width=width) diff --git a/src/sqlbuild/presentation/main/summary_footer.py b/src/sqlbuild/presentation/main/summary_footer.py deleted file mode 100644 index 5c7c2ee6f0..0000000000 --- a/src/sqlbuild/presentation/main/summary_footer.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Public status summary footer formatting entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.summary_footer import ( - format_summary_footer as _format_summary_footer, -) - - -def format_summary_footer( - *, counts: tuple[tuple[str, int], ...], use_color: bool, elapsed: str | None = None -) -> str: - """Format summary counts with semantic colors.""" - - return _format_summary_footer(counts=counts, use_color=use_color, elapsed=elapsed) diff --git a/src/sqlbuild/presentation/main/visible_entries.py b/src/sqlbuild/presentation/main/visible_entries.py deleted file mode 100644 index 161b58f638..0000000000 --- a/src/sqlbuild/presentation/main/visible_entries.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public bounded-output visibility entry.""" - -from __future__ import annotations - -from collections.abc import Sequence - -from sqlbuild.presentation._helpers.display import visible_entries as _visible_entries -from sqlbuild.presentation.models import DisplayOptions - - -def visible_entries[T](*, entries: Sequence[T], options: DisplayOptions) -> Sequence[T]: - """Return entries visible under the current display cap.""" - - return _visible_entries(entries=entries, options=options) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/config.py b/src/sqlbuild/rule_engine/_helpers/engine/config.py index e69c2e8a3e..383f54139b 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/config.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/config.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import cast +from sqlbuild._native import format_count_noun from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME @@ -14,7 +15,6 @@ from sqlbuild.errors.setting_help.main.join_helps import join_helps from sqlbuild.errors.setting_help.main.setting_help import setting_help from sqlbuild.errors.setting_help.main.setting_note import setting_note -from sqlbuild.presentation.main.count_noun import format_count_noun from sqlbuild.rule_engine._helpers.engine.native import load_native_config from sqlbuild.rule_engine.constants import ( ALLOW_EXCEPTIONS_KEY, diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py deleted file mode 100644 index e91ac52abc..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py +++ /dev/null @@ -1,558 +0,0 @@ -"""Static custom-rule test evidence and source fingerprinting.""" - -from __future__ import annotations - -import ast -import hashlib -import inspect -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from sqlbuild.rule_engine.models import Rule - -_PUBLIC_HARNESS_MODULES: frozenset[str] = frozenset({"sqlbuild.rules", "sqlbuild.rules.testing"}) -_INIT_MODULE_NAME: str = "__init__" -_RULES_DIRECTORY_NAME: str = "rules" -_STAR_IMPORT_NAME: str = "*" -_MIN_PARAMETRIZE_ARGUMENTS: int = 2 -_PARAMETRIZE_ATTRIBUTE: str = "parametrize" -_PYTEST_MARK_ATTRIBUTE: str = "mark" -_PYTEST_MODULE_NAME: str = "pytest" - - -@dataclass(frozen=True, order=True) -class _RuleTestEvidence: - path: str - line: int - column: int - owner: str - - -def custom_rule_test_evidence(*, rule: Rule, project_dir: Path) -> tuple[_RuleTestEvidence, ...]: - """Return statically associated public-harness cases for one exact custom rule.""" - - if rule.source is None: - return () - source_path: Path = Path(rule.source).resolve() - rule_module_names: frozenset[str] = _module_names(path=source_path, project_dir=project_dir) - check_name: str = getattr(rule.check, "__name__", "") - evidence: set[_RuleTestEvidence] = set() - for path in sorted(project_dir.glob("tests/**/*.py")): - if not (path.name.startswith("test_") or path.name.endswith("_test.py")): - continue - evidence.update( - _file_evidence( - path=path, - project_dir=project_dir, - rule_module_names=rule_module_names, - check_name=check_name, - ) - ) - return tuple(sorted(evidence)) - - -def custom_rule_import_closure(*, rule: Rule, project_dir: Path) -> tuple[Path, ...]: - """Return repository-owned helper files imported by one rule module.""" - - if rule.source is None: - return () - return _import_closure( - source_path=Path(rule.source).resolve(), project_dir=project_dir.resolve() - ) - - -def custom_rule_implementation_fingerprint( - *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] | None = None -) -> str: - """Fingerprint one rule's whole source file and its repository-owned imported helper closure.""" - - digest: Any = hashlib.sha256() - if rule.source is None: - digest.update(inspect.getsource(rule.check).encode()) - return digest.hexdigest() - root: Path = project_dir.resolve() - source_path: Path = Path(rule.source).resolve() - closure: tuple[Path, ...] = ( - custom_rule_import_closure(rule=rule, project_dir=root) - if import_closure is None - else import_closure - ) - for path in (source_path, *closure): - digest.update(_fingerprint_label(path=path, root=root).encode()) - digest.update(b"\0") - digest.update(path.read_bytes()) - return digest.hexdigest() - - -def _fingerprint_label(*, path: Path, root: Path) -> str: - return path.relative_to(root).as_posix() if path.is_relative_to(root) else path.as_posix() - - -def _import_closure(*, source_path: Path, project_dir: Path) -> tuple[Path, ...]: - rules_root: Path = (project_dir / _RULES_DIRECTORY_NAME).resolve() - pending: list[Path] = [source_path] - visited: set[Path] = {source_path} - helpers: set[Path] = set() - while pending: - current: Path = pending.pop() - candidates: set[Path] = set(_package_inits(path=current, rules_root=rules_root)) - try: - tree: ast.Module = ast.parse(current.read_text(encoding="utf-8"), filename=str(current)) - except (OSError, UnicodeError, SyntaxError): - tree = ast.Module(body=[], type_ignores=[]) - for node in ast.walk(tree): - candidates.update( - _local_import_paths(node=node, source_path=current, rules_root=rules_root) - ) - for candidate in candidates - visited: - visited.add(candidate) - helpers.add(candidate) - pending.append(candidate) - return tuple(sorted(helpers)) - - -def _local_import_paths(*, node: ast.AST, source_path: Path, rules_root: Path) -> tuple[Path, ...]: - """Resolve every repository-owned rules module one import statement can execute.""" - - targets: list[tuple[Path, tuple[str, ...]]] = [] - if isinstance(node, ast.Import): - targets.extend( - (rules_root.parent, tuple(alias.name.split("."))) - for alias in node.names - if alias.name.split(".")[0] == _RULES_DIRECTORY_NAME - ) - elif isinstance(node, ast.ImportFrom): - base: Path = rules_root.parent - module_parts: tuple[str, ...] = tuple((node.module or "").split(".")) if node.module else () - if node.level: - base = source_path.parent - for _ in range(node.level - 1): - base = base.parent - elif not module_parts or module_parts[0] != _RULES_DIRECTORY_NAME: - return () - targets.append((base, module_parts)) - targets.extend( - (base, (*module_parts, alias.name)) - for alias in node.names - if alias.name != _STAR_IMPORT_NAME - ) - resolved: list[Path] = [] - for base, parts in targets: - module: Path | None = _module_file(base=base, parts=parts, rules_root=rules_root) - if module is not None: - resolved.append(module) - resolved.extend(_package_inits(path=module, rules_root=rules_root)) - return tuple(resolved) - - -def _module_file(*, base: Path, parts: tuple[str, ...], rules_root: Path) -> Path | None: - candidate: Path = base.joinpath(*parts) - for path in (candidate / "__init__.py", candidate.with_suffix(".py") if parts else None): - if path is not None and path.is_file(): - resolved: Path = path.resolve() - return resolved if resolved.is_relative_to(rules_root) else None - return None - - -def _package_inits(*, path: Path, rules_root: Path) -> tuple[Path, ...]: - """Return package initializers executed before one module under the rules root.""" - - inits: list[Path] = [] - for parent in path.parents: - if not parent.is_relative_to(rules_root): - break - init: Path = parent / "__init__.py" - if not init.is_file(): - continue - resolved: Path = init.resolve() - if resolved != path and resolved.is_relative_to(rules_root): - inits.append(resolved) - return tuple(inits) - - -def _file_evidence( - *, path: Path, project_dir: Path, rule_module_names: frozenset[str], check_name: str -) -> set[_RuleTestEvidence]: - try: - tree: ast.Module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except (OSError, UnicodeError, SyntaxError): - return set() - imports: dict[str, tuple[str, str | None]] = _imports(tree=tree) - assignments: dict[str, ast.expr] = _assignments(tree.body) - constructors: frozenset[str] = _constructors(tree=tree, imports=imports) - parents: dict[ast.AST, ast.AST] = {} - for parent in ast.walk(tree): - for child in ast.iter_child_nodes(parent): - parents[child] = parent - relative_path: str = path.relative_to(project_dir).as_posix() - found: set[_RuleTestEvidence] = set() - for node in ast.walk(tree): - if not isinstance(node, ast.Call) or not _is_imported_symbol( - expression=node.func, - imports=imports, - modules=_PUBLIC_HARNESS_MODULES, - symbol="evaluate_rule", - ): - continue - owner: ast.FunctionDef | ast.AsyncFunctionDef | None = _test_owner( - node=node, parents=parents - ) - if ( - owner is None - or _is_statically_dead(node=node, owner=owner, parents=parents) - or _is_statically_skipped(owner=owner, imports=imports, parents=parents) - ): - continue - rule_argument: ast.expr | None = _keyword(call=node, name="rule") - case_argument: ast.expr | None = _keyword(call=node, name="test_case") - if ( - rule_argument is None - or case_argument is None - or not _is_imported_symbol( - expression=rule_argument, - imports=imports, - modules=rule_module_names, - symbol=check_name, - ) - ): - continue - owner_name: str = f"{relative_path}:{owner.name}" - for case in _case_declarations( - expression=case_argument, - owner=owner, - imports=imports, - assignments=assignments, - constructors=constructors, - ): - found.add( - _RuleTestEvidence( - path=relative_path, - line=case.lineno, - column=case.col_offset + 1, - owner=owner_name, - ) - ) - return found - - -def _case_declarations( - *, - expression: ast.expr, - owner: ast.FunctionDef | ast.AsyncFunctionDef, - imports: dict[str, tuple[str, str | None]], - assignments: dict[str, ast.expr], - constructors: frozenset[str], -) -> tuple[ast.Call, ...]: - direct: tuple[ast.Call, ...] = _declared_cases( - expression=expression, - field=None, - imports=imports, - assignments=assignments, - constructors=constructors, - seen=frozenset(), - ) - if direct: - return direct - parameter: str | None = None - field: str | None = None - if isinstance(expression, ast.Name): - parameter = expression.id - elif isinstance(expression, ast.Attribute) and isinstance(expression.value, ast.Name): - parameter = expression.value.id - field = expression.attr - if parameter is None: - return () - values: ast.expr | None = _parametrized_values( - owner=owner, parameter=parameter, imports=imports - ) - if values is None: - return () - return _declared_cases( - expression=values, - field=field, - imports=imports, - assignments=assignments, - constructors=constructors, - seen=frozenset(), - ) - - -def _declared_cases( - *, - expression: ast.expr, - field: str | None, - imports: dict[str, tuple[str, str | None]], - assignments: dict[str, ast.expr], - constructors: frozenset[str], - seen: frozenset[str], -) -> tuple[ast.Call, ...]: - if ( - isinstance(expression, ast.Name) - and expression.id in assignments - and expression.id not in seen - ): - return _declared_cases( - expression=assignments[expression.id], - field=field, - imports=imports, - assignments=assignments, - constructors=constructors, - seen=seen | {expression.id}, - ) - if isinstance(expression, (ast.List, ast.Tuple, ast.Set)): - cases: list[ast.Call] = [] - for item in expression.elts: - cases.extend( - _declared_cases( - expression=item, - field=field, - imports=imports, - assignments=assignments, - constructors=constructors, - seen=seen, - ) - ) - return tuple(cases) - if not isinstance(expression, ast.Call): - return () - if field is None and _is_imported_symbol( - expression=expression.func, - imports=imports, - modules=_PUBLIC_HARNESS_MODULES, - symbol="RuleCase", - ): - return (expression,) - if field is None: - return () - if not isinstance(expression.func, ast.Name) or expression.func.id not in constructors: - return () - value: ast.expr | None = next( - (keyword.value for keyword in expression.keywords if keyword.arg == field), None - ) - if value is None: - return () - return _declared_cases( - expression=value, - field=None, - imports=imports, - assignments=assignments, - constructors=constructors, - seen=seen, - ) - - -def _parametrized_values( - *, - owner: ast.FunctionDef | ast.AsyncFunctionDef, - parameter: str, - imports: dict[str, tuple[str, str | None]], -) -> ast.expr | None: - for decorator in owner.decorator_list: - if not isinstance(decorator, ast.Call) or not _is_parametrize_call( - expression=decorator.func, imports=imports - ): - continue - if len(decorator.args) < _MIN_PARAMETRIZE_ARGUMENTS or not _parameter_names( - decorator.args[0] - ) == (parameter,): - continue - return decorator.args[1] - return None - - -def _imports(tree: ast.Module) -> dict[str, tuple[str, str | None]]: - result: dict[str, tuple[str, str | None]] = {} - for node in tree.body: - if isinstance(node, ast.Import): - for alias in node.names: - result[alias.asname or alias.name.split(".")[0]] = (alias.name, None) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module is not None: - for alias in node.names: - result[alias.asname or alias.name] = (node.module, alias.name) - return result - - -def _assignments(statements: list[ast.stmt]) -> dict[str, ast.expr]: - result: dict[str, ast.expr] = {} - for statement in statements: - if isinstance(statement, (ast.Assign, ast.AnnAssign)): - value: ast.expr | None = statement.value - targets: list[ast.expr] = ( - statement.targets if isinstance(statement, ast.Assign) else [statement.target] - ) - if value is not None: - for target in targets: - if isinstance(target, ast.Name): - result[target.id] = value - return result - - -def _constructors( - *, tree: ast.Module, imports: dict[str, tuple[str, str | None]] -) -> frozenset[str]: - declared: set[str] = {node.name for node in tree.body if isinstance(node, ast.ClassDef)} - declared.update(name for name, (_, symbol) in imports.items() if symbol is not None) - return frozenset(declared) - - -def _is_imported_symbol( - *, - expression: ast.expr, - imports: dict[str, tuple[str, str | None]], - modules: frozenset[str], - symbol: str, -) -> bool: - if isinstance(expression, ast.Name): - imported: tuple[str, str | None] | None = imports.get(expression.id) - return imported is not None and imported[0] in modules and imported[1] == symbol - if isinstance(expression, ast.Attribute) and isinstance(expression.value, ast.Name): - imported = imports.get(expression.value.id) - if imported is None or expression.attr != symbol: - return False - module, imported_name = imported - qualified_module: str = module if imported_name is None else f"{module}.{imported_name}" - return qualified_module in modules - return False - - -def _test_owner( - *, node: ast.AST, parents: dict[ast.AST, ast.AST] -) -> ast.FunctionDef | ast.AsyncFunctionDef | None: - current: ast.AST = node - while current in parents: - current = parents[current] - if isinstance(current, (ast.Lambda, ast.ClassDef)): - return None - if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef)): - if not current.name.startswith("test_"): - return None - parent: ast.AST | None = parents.get(current) - if isinstance(parent, ast.Module): - return current - if ( - isinstance(parent, ast.ClassDef) - and parent.name.startswith("Test") - and isinstance(parents.get(parent), ast.Module) - ): - return current - return None - return None - - -def _is_statically_skipped( - *, - owner: ast.FunctionDef | ast.AsyncFunctionDef, - imports: dict[str, tuple[str, str | None]], - parents: dict[ast.AST, ast.AST], -) -> bool: - declarations: list[ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef] = [owner] - parent: ast.AST | None = parents.get(owner) - if isinstance(parent, ast.ClassDef): - declarations.append(parent) - for declaration in declarations: - for decorator in declaration.decorator_list: - target: ast.expr = decorator.func if isinstance(decorator, ast.Call) else decorator - if _is_pytest_mark(expression=target, imports=imports, mark="skip"): - return True - if not isinstance(decorator, ast.Call) or not _is_pytest_mark( - expression=decorator.func, imports=imports, mark="skipif" - ): - continue - condition: ast.expr | None = ( - decorator.args[0] if decorator.args else _keyword(call=decorator, name="condition") - ) - if isinstance(condition, ast.Constant) and condition.value is True: - return True - return False - - -def _is_statically_dead( - *, - node: ast.AST, - owner: ast.FunctionDef | ast.AsyncFunctionDef, - parents: dict[ast.AST, ast.AST], -) -> bool: - current: ast.AST = node - while current is not owner: - parent: ast.AST | None = parents.get(current) - if parent is None: - return True - if isinstance(parent, (ast.If, ast.While)) and isinstance(parent.test, ast.Constant): - in_body: bool = current in parent.body - if (in_body and not bool(parent.test.value)) or ( - not in_body and bool(parent.test.value) - ): - return True - current = parent - return False - - -def _keyword(*, call: ast.Call, name: str) -> ast.expr | None: - return next((keyword.value for keyword in call.keywords if keyword.arg == name), None) - - -def _is_parametrize_call( - *, expression: ast.expr, imports: dict[str, tuple[str, str | None]] -) -> bool: - return ( - isinstance(expression, ast.Attribute) - and expression.attr == _PARAMETRIZE_ATTRIBUTE - and isinstance(expression.value, ast.Attribute) - and expression.value.attr == _PYTEST_MARK_ATTRIBUTE - and isinstance(expression.value.value, ast.Name) - and imports.get(expression.value.value.id) == (_PYTEST_MODULE_NAME, None) - ) - - -def _is_pytest_mark( - *, expression: ast.expr, imports: dict[str, tuple[str, str | None]], mark: str -) -> bool: - return ( - isinstance(expression, ast.Attribute) - and expression.attr == mark - and isinstance(expression.value, ast.Attribute) - and expression.value.attr == _PYTEST_MARK_ATTRIBUTE - and isinstance(expression.value.value, ast.Name) - and imports.get(expression.value.value.id) == (_PYTEST_MODULE_NAME, None) - ) - - -def _parameter_names(expression: ast.expr) -> tuple[str, ...]: - if isinstance(expression, ast.Constant) and isinstance(expression.value, str): - return tuple(name.strip() for name in expression.value.split(",")) - if isinstance(expression, (ast.List, ast.Tuple)): - values: list[str] = [] - for item in expression.elts: - if not isinstance(item, ast.Constant) or not isinstance(item.value, str): - return () - values.append(item.value) - return tuple(values) - return () - - -def _module_names(*, path: Path, project_dir: Path) -> frozenset[str]: - relative: Path = path.relative_to(project_dir.resolve()).with_suffix("") - parts: tuple[str, ...] = relative.parts - if parts and parts[-1] == _INIT_MODULE_NAME: - parts = parts[:-1] - names: set[str] = {".".join(parts)} - for index in range(len(parts)): - package_parts: tuple[str, ...] = parts[index:-1] - package_root: Path = project_dir.joinpath(*parts[:index]) - if package_parts and all( - (package_root.joinpath(*package_parts[:offset], "__init__.py")).is_file() - for offset in range(1, len(package_parts) + 1) - ): - names.add(".".join(parts[index:])) - return frozenset(names) - - -def _source_root(*, source_path: Path, project_dir: Path) -> Path: - root: Path = project_dir.resolve() - for parent in (source_path.parent, *source_path.parents): - if parent == root: - break - if parent.name == _RULES_DIRECTORY_NAME: - return parent - return source_path.parent diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index e6805a87d2..fd3d9d5d8e 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -15,15 +15,13 @@ from sqlbuild._native import ( compiled_project_record, + custom_rule_implementation_fingerprint, + custom_rule_import_closure, ) from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( - custom_rule_implementation_fingerprint, - custom_rule_import_closure, -) from sqlbuild.rule_engine._helpers.engine.fact_replay import full_fact_keys, is_tree_fact from sqlbuild.rule_engine._helpers.host.custom_host_pool import ( decode_readset, diff --git a/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py b/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py index b7fc4c16e4..22173a8d50 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py @@ -5,7 +5,7 @@ import ast from pathlib import Path -from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import custom_rule_import_closure +from sqlbuild._native import custom_rule_import_closure from sqlbuild.rule_engine.constants import CUSTOM_RULE_IMPORT_ROOTS from sqlbuild.rule_engine.exceptions import NonHermeticRuleError, RulesError from sqlbuild.rule_engine.models import Rule diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 7c8ae7d235..d50dc4b254 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -11,13 +11,13 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( +from sqlbuild._native import ( custom_rule_implementation_fingerprint, custom_rule_import_closure, custom_rule_test_evidence, ) +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.run.native_rows import ( build_rules_request, diff --git a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py b/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py index 5c3699b769..5ff0195d61 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py @@ -7,12 +7,12 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ( + custom_rule_implementation_fingerprint, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors -from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( - custom_rule_implementation_fingerprint, -) from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules from sqlbuild.rule_engine._helpers.engine.native import evaluate_native from sqlbuild.rule_engine.models import ResolvedRuleset, Rule, RulesConfig, RulesResult diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py b/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py index 2ac99ed591..156d4b925f 100644 --- a/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py +++ b/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py @@ -2,71 +2,12 @@ from __future__ import annotations -import json - from sqlbuild.rule_engine._helpers.guidance.thresholds import format_threshold_lines -from sqlbuild.rule_engine.constants import RULE_FIX_AVAILABLE_NOTE -from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult +from sqlbuild.rule_engine.models import Rule, RulesConfig _MINIMUM_RULES_CODES: frozenset[str] = frozenset({"SQBRTEST201", "SQBRTEST202"}) -def format_result_text(*, result: RulesResult) -> str: - if not result.findings: - return ( - f"Rules passed: {result.evaluated_models} models evaluated, 0 findings " - f"({result.cache_hits} cache hits, {result.cache_misses} misses)" - ) - blocks: list[str] = [] - for finding in result.findings: - blocks.append( - f"{finding.path}:{finding.line}:{finding.column} " - f"[{finding.code}] {finding.message}\n" - f" Remediation: {finding.remediation}" - ) - if finding.affected_rules: - blocks.append(f" Affected Rules: {', '.join(finding.affected_rules)}") - if finding.fixable: - blocks.append(f" {RULE_FIX_AVAILABLE_NOTE}") - blocks.append(f"Found {len(result.findings)} Rules findings") - if result.unevaluated_resources: - blocks.append( - f"{result.evaluated_models} models evaluated, " - f"{result.unevaluated_resources} resources could not be evaluated" - ) - return "\n".join(blocks) - - -def format_result_json(*, result: RulesResult) -> str: - findings: list[dict[str, object]] = [] - for finding in result.findings: - findings.append( - { - "code": finding.code, - "path": finding.path.as_posix(), - "line": finding.line, - "column": finding.column, - "message": finding.message, - "remediation": finding.remediation, - **( - {"affected_rules": list(finding.affected_rules)} - if finding.affected_rules - else {} - ), - **({"fixable": True} if finding.fixable else {}), - } - ) - payload: dict[str, object] = { - "evaluated_models": result.evaluated_models, - "unevaluated_resources": result.unevaluated_resources, - "cache_hits": result.cache_hits, - "cache_misses": result.cache_misses, - "finding_count": len(result.findings), - "findings": findings, - } - return json.dumps(payload, indent=2, sort_keys=True) - - def format_rule_text(*, rule: Rule, config: RulesConfig | None = None) -> str: lines: list[str] = [ f"{rule.code}: {rule.slug}", diff --git a/src/sqlbuild/rule_engine/_helpers/run/literal_duplicates.py b/src/sqlbuild/rule_engine/_helpers/run/literal_duplicates.py deleted file mode 100644 index 7212cade71..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/literal_duplicates.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Point repeated long literals at one shared constant in the nearest common folder.""" - -from __future__ import annotations - -import os -from dataclasses import replace -from pathlib import Path, PurePosixPath - -from sqlbuild.rule_engine.models import Finding - -_LONG_LITERAL_CODE: str = "SQBRSQL044" -_QUOTE: str = "'" -_LISTED_PATHS: int = 5 -_MIN_SHARED_FILES: int = 2 -_MIN_SCOPED_FOLDER_DEPTH: int = 2 -_DECLARATION_GROUP: str = "_sqlbuild" -_LOCAL_CONSTANTS: str = "_sqlbuild/_constants/" -_INHERITED_CONSTANTS: str = "_sqlbuild/constants/" -_PROJECT_CONSTANTS: str = "constants/" - - -def with_duplicate_literal_hints( - *, findings: tuple[Finding, ...], project_dir: Path -) -> tuple[Finding, ...]: - """Prefix SQBRSQL044 help with the other files that repeat the identical literal.""" - - texts: dict[Path, list[str]] = {} - literals: dict[int, str] = {} - for index, finding in enumerate(findings): - if finding.code != _LONG_LITERAL_CODE or finding.unevaluated: - continue - literal: str | None = _literal_at( - lines=_lines(texts=texts, path=project_dir / finding.path), - line=finding.line, - column=finding.column, - ) - if literal is not None: - literals[index] = literal - paths_by_literal: dict[str, set[PurePosixPath]] = {} - for index, literal in literals.items(): - paths_by_literal.setdefault(literal, set()).add(PurePosixPath(findings[index].path)) - hinted: list[Finding] = list(findings) - for index, literal in literals.items(): - paths: set[PurePosixPath] = paths_by_literal[literal] - if len(paths) < _MIN_SHARED_FILES: - continue - own: PurePosixPath = PurePosixPath(findings[index].path) - hint: str = _hint(others=sorted(paths - {own}), every=paths) - hinted[index] = replace( - findings[index], remediation=f"{hint} {findings[index].remediation}" - ) - return tuple(hinted) - - -def _lines(*, texts: dict[Path, list[str]], path: Path) -> list[str]: - if path not in texts: - try: - texts[path] = path.read_text(encoding="utf-8").splitlines() - except OSError: - texts[path] = [] - return texts[path] - - -def _literal_at(*, lines: list[str], line: int, column: int) -> str | None: - """The single-quoted literal starting at a 1-based position, quotes included.""" - - if not 1 <= line <= len(lines): - return None - text: str = lines[line - 1] - start: int = column - 1 - if start < 0 or text[start : start + 1] != _QUOTE: - return None - position: int = start + 1 - while position < len(text): - if text[position] == _QUOTE: - if text[position + 1 : position + 2] == _QUOTE: - position += 2 - continue - return text[start : position + 1] - position += 1 - return None - - -def _hint(*, others: list[PurePosixPath], every: set[PurePosixPath]) -> str: - listed: str = ", ".join(path.as_posix() for path in others[:_LISTED_PATHS]) - more: str = f" and {len(others) - _LISTED_PATHS} more" if len(others) > _LISTED_PATHS else "" - owners: set[PurePosixPath] = {_owner(path) for path in every} - common: PurePosixPath = _common_folder(owners) - if len(common.parts) < _MIN_SCOPED_FOLDER_DEPTH: - placement: str = ( - f"no folder below the project roots holds every use, so declare it once in " - f"top-level `{_PROJECT_CONSTANTS}`" - ) - else: - directory: str = _LOCAL_CONSTANTS if owners == {common} else _INHERITED_CONSTANTS - placement = ( - f"their nearest common folder is `{common.as_posix()}`, so declare it once in " - f"`{common.as_posix()}/{directory}`" - ) - return ( - f"The identical literal also appears in {listed}{more}; {placement} " - "and reference it with `@const` in each file." - ) - - -def _owner(path: PurePosixPath) -> PurePosixPath: - """The folder whose declarations a file sees: above `_sqlbuild/`, else its own folder.""" - - if _DECLARATION_GROUP in path.parts: - return PurePosixPath(*path.parts[: path.parts.index(_DECLARATION_GROUP)]) - return path.parent - - -def _common_folder(folders: set[PurePosixPath]) -> PurePosixPath: - return PurePosixPath(os.path.commonpath([folder.as_posix() for folder in folders])) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 9f3710fcd6..cb50216345 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -16,6 +16,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( project_with, + with_duplicate_literal_hints, ) from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.models import ( @@ -63,7 +64,6 @@ ) from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings -from sqlbuild.rule_engine._helpers.run.literal_duplicates import with_duplicate_literal_hints from sqlbuild.rule_engine._helpers.run.memory import release_freed_memory from sqlbuild.rule_engine.constants import SQL_RULES_CACHE_FILE, TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.exceptions import RulesError diff --git a/src/sqlbuild/rule_engine/main/render_result.py b/src/sqlbuild/rule_engine/main/render_result.py index ca99d8ecfb..bb6afd1c1c 100644 --- a/src/sqlbuild/rule_engine/main/render_result.py +++ b/src/sqlbuild/rule_engine/main/render_result.py @@ -1,15 +1,10 @@ """Rules result formatting entrypoint.""" -from sqlbuild.rule_engine._helpers.guidance.presentation import ( - format_result_json, - format_result_text, -) +from sqlbuild._native import format_rules_result from sqlbuild.rule_engine.models import RulesResult def format_result(*, result: RulesResult, json_output: bool) -> str: """Format one rules result for the requested CLI surface.""" - if json_output: - return format_result_json(result=result) - return format_result_text(result=result) + return format_rules_result(result=result, json_output=json_output) diff --git a/src/sqlbuild/spec/contracts/_helpers/config_values.py b/src/sqlbuild/spec/contracts/_helpers/config_values.py deleted file mode 100644 index e9c1c75715..0000000000 --- a/src/sqlbuild/spec/contracts/_helpers/config_values.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Typed extraction from raw authored configuration mappings.""" - -from __future__ import annotations - -from collections.abc import Mapping -from datetime import date, datetime -from typing import Never, cast - -from sqlbuild.spec.contracts.exceptions import ConfigValueTypeError - - -def _get_config_str(*, values: Mapping[str, object], key: str) -> str | None: - """Return an authored string, preserving a present empty string.""" - - value: object | None = values.get(key) - if value is None: - return None - if not isinstance(value, str): - _raise_wrong_type(key=key, expected="a string", value=value) - return value - - -def _get_config_bool(*, values: Mapping[str, object], key: str) -> bool | None: - """Return an authored boolean, or None when the key is absent or None.""" - - value: object | None = values.get(key) - if value is None: - return None - if not isinstance(value, bool): - _raise_wrong_type(key=key, expected="a boolean", value=value) - return value - - -def _get_config_string_tuple(*, values: Mapping[str, object], key: str) -> tuple[str, ...] | None: - """Return an authored sequence of strings, or None when absent or None.""" - - value: object | None = values.get(key) - if value is None: - return None - if not isinstance(value, list | tuple) or not all(isinstance(item, str) for item in value): - _raise_wrong_type(key=key, expected="a list or tuple of strings", value=value) - return cast(tuple[str, ...], tuple(value)) - - -def _get_config_int(*, values: Mapping[str, object], key: str) -> int | None: - """Return an authored integer, excluding booleans, or None when absent or None.""" - - value: object | None = values.get(key) - if value is None: - return None - if isinstance(value, bool) or not isinstance(value, int): - _raise_wrong_type(key=key, expected="an integer", value=value) - return value - - -def _get_config_cursor_bound(*, values: Mapping[str, object], key: str) -> str | None: - """Return a cursor bound normalized to text from its supported authored scalar types.""" - - value: object | None = values.get(key) - if value is None: - return None - if isinstance(value, bool) or not isinstance(value, str | int | date | datetime): - _raise_wrong_type( - key=key, - expected="a string, integer, date, or datetime", - value=value, - ) - return value.isoformat() if isinstance(value, date | datetime) else str(value) - - -def _raise_wrong_type(*, key: str, expected: str, value: object | None) -> Never: - raise ConfigValueTypeError(key=key, expected=expected, actual_type=type(value)) diff --git a/src/sqlbuild/spec/contracts/main/get_config_bool.py b/src/sqlbuild/spec/contracts/main/get_config_bool.py deleted file mode 100644 index b2e30d8be5..0000000000 --- a/src/sqlbuild/spec/contracts/main/get_config_bool.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public typed boolean config extraction.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.spec.contracts._helpers.config_values import _get_config_bool - - -def get_config_bool(*, values: Mapping[str, object], key: str) -> bool | None: - """Return an authored boolean, or None only when the key is absent.""" - - return _get_config_bool(values=values, key=key) diff --git a/src/sqlbuild/spec/contracts/main/get_config_cursor_bound.py b/src/sqlbuild/spec/contracts/main/get_config_cursor_bound.py deleted file mode 100644 index 781b74e7ff..0000000000 --- a/src/sqlbuild/spec/contracts/main/get_config_cursor_bound.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public cursor-bound config extraction.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.spec.contracts._helpers.config_values import _get_config_cursor_bound - - -def get_config_cursor_bound(*, values: Mapping[str, object], key: str) -> str | None: - """Return a cursor bound normalized from its supported authored scalar types.""" - - return _get_config_cursor_bound(values=values, key=key) diff --git a/src/sqlbuild/spec/contracts/main/get_config_int.py b/src/sqlbuild/spec/contracts/main/get_config_int.py deleted file mode 100644 index bc4224651a..0000000000 --- a/src/sqlbuild/spec/contracts/main/get_config_int.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public typed integer config extraction.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.spec.contracts._helpers.config_values import _get_config_int - - -def get_config_int(*, values: Mapping[str, object], key: str) -> int | None: - """Return an authored integer, excluding booleans, or None when absent.""" - - return _get_config_int(values=values, key=key) diff --git a/src/sqlbuild/spec/contracts/main/get_config_str.py b/src/sqlbuild/spec/contracts/main/get_config_str.py deleted file mode 100644 index da11614572..0000000000 --- a/src/sqlbuild/spec/contracts/main/get_config_str.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public typed string config extraction.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.spec.contracts._helpers.config_values import _get_config_str - - -def get_config_str(*, values: Mapping[str, object], key: str) -> str | None: - """Return an authored string, preserving a present empty string.""" - - return _get_config_str(values=values, key=key) diff --git a/src/sqlbuild/spec/contracts/main/get_config_string_tuple.py b/src/sqlbuild/spec/contracts/main/get_config_string_tuple.py deleted file mode 100644 index d86cc04a7a..0000000000 --- a/src/sqlbuild/spec/contracts/main/get_config_string_tuple.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public typed string-sequence config extraction.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.spec.contracts._helpers.config_values import _get_config_string_tuple - - -def get_config_string_tuple(*, values: Mapping[str, object], key: str) -> tuple[str, ...] | None: - """Return an authored sequence of strings, or None when absent.""" - - return _get_config_string_tuple(values=values, key=key) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_inspection_command_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_inspection_command_performance.py index 2d4705fbba..955f164706 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_inspection_command_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_inspection_command_performance.py @@ -37,7 +37,7 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) _MIB: int = 1024 * 1024 _MODEL_COUNT: int = INSPECTION_BENCHMARK_MODEL_COUNT -_MODEL_LINEAGE_MAX_RSS_BYTES: int = 144 * _MIB +_MODEL_LINEAGE_MAX_RSS_BYTES: int = 176 * _MIB @pytest.mark.performance diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py index 7c1363175b..023e02cbbc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.cli.commands.main.workspace._playground import run_playground +from sqlbuild._native import run_playground_command as run_playground from sqlbuild.cli.commands.models import PlaygroundCommandRequest from tests.e2e.src.sqlbuild.cli.commands.main.playground._test_types import ( LegacyCodePageOutputTestCase, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py index a0376e58ec..f2c1d042cc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.cli.commands.main.workspace._playground import run_playground +from sqlbuild._native import run_playground_command as run_playground from sqlbuild.cli.commands.models import PlaygroundCommandRequest from tests.e2e.src.sqlbuild.cli.commands.main.playground._test_types import ( PlaygroundCompileBuildTestCase, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/skills/test_skills.py b/tests/e2e/src/sqlbuild/cli/commands/main/skills/test_skills.py index c964a8cfee..636a0903d0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/skills/test_skills.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/skills/test_skills.py @@ -5,7 +5,6 @@ import pytest from sqlbuild._native import run_cli as main -from sqlbuild.cli.commands._helpers.skills.update import generated_marker from tests.e2e.src.sqlbuild.cli.commands.main.skills._test_types import ( SkillsCliOverwriteTestCase, SkillsCliTestCase, @@ -15,6 +14,8 @@ write_existing_skill_file, ) +GENERATED_MARKER: str = "" + @pytest.mark.parametrize( "test_case", @@ -55,7 +56,7 @@ def test_given_skills_update_cli_when_running_then_writes_expected_skill_files( for expected_file in test_case.expected_files: contents: str = (tmp_path / expected_file).read_text(encoding="utf-8") assert contents.startswith("---\nname: sqlbuild\n") - assert generated_marker in contents + assert GENERATED_MARKER in contents assert test_case.expected_content_fragment in contents assert ( existing_file_paths(project_dir=tmp_path, relative_paths=test_case.unexpected_files) == () diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py index 817c9a5bd4..a25a7ae59a 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py @@ -21,7 +21,7 @@ materialize, ) -from sqlbuild.cli.commands.main.workspace._playground import run_playground +from sqlbuild._native import run_playground_command as run_playground from sqlbuild.cli.commands.models import PlaygroundCommandRequest from sqlbuild.integrations.dagster import ( SqlBuildCliResource, diff --git a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py index ae77782cf8..64e2a2854d 100644 --- a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py @@ -11,7 +11,7 @@ import pytest -from sqlbuild.cli.commands.main.workspace._playground import run_playground +from sqlbuild._native import run_playground_command as run_playground from sqlbuild.cli.commands.models import PlaygroundCommandRequest from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets from sqlbuild.integrations.rivers._helpers.assets import build_asset_defs diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py index db5666b4cf..efd1bba37a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.cli.commands._helpers.refactor.command import run_refactor_command +from sqlbuild._native import run_refactor_command from sqlbuild.cli.commands.models import RefactorCommandRequest from sqlbuild.cli.commands.types import CliCommand from tests.integration.src.sqlbuild.cli.commands._helpers.refactor._test_types import ( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py index b01f9eadeb..e5df8a4dc9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py @@ -91,9 +91,7 @@ def test_given_legacy_model_imports_when_loading_focused_models_then_exports_kee "CliEntrypointHandlers": entry_models, "ParsedCliInvocation": entry_models, "SelectorFileSummary": entry_models, - "SkillInstallTarget": output_models, "SkillMaintenanceResult": output_models, - "SkillSettings": output_models, "SkillUpdateResult": output_models, "WrittenTarget": output_models, } diff --git a/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py b/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py index 07ef335f9f..c1b903e346 100644 --- a/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py +++ b/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py @@ -2,10 +2,11 @@ from __future__ import annotations +from pathlib import Path + import pytest -from sqlbuild.cli.output._helpers.integration_result import build_integration_result -from sqlbuild.cli.output.models import IntegrationCheckResult, IntegrationResultEnvelope +from sqlbuild._native import IntegrationCheckResult, IntegrationResultEnvelope from sqlbuild.compiler.auditing.models import ( MeasurementThresholdBound, MeasurementThresholds, @@ -20,6 +21,7 @@ from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.runtime.observability.models import LifecycleEvent from tests.integration.src.sqlbuild.cli.output._test_types import MeasurementEnvelopeTestCase +from tests.unit.src.sqlbuild.cli.output._helpers.helpers import written_integration_record from tests.unit.src.sqlbuild.runtime.observability.helpers import lifecycle_event @@ -36,7 +38,7 @@ ids=lambda case: case.description, ) def test_given_measurement_execution_result_when_building_envelope_then_decode_round_trip_is_complete( - test_case: MeasurementEnvelopeTestCase, + test_case: MeasurementEnvelopeTestCase, tmp_path: Path ) -> None: result: AuditExecutionResult = AuditExecutionResult( audit_name="valid_rate", @@ -67,15 +69,10 @@ def test_given_measurement_execution_result_when_building_envelope_then_decode_r }, ) - envelope: IntegrationResultEnvelope | None = build_integration_result( - result=result, - terminal=terminal, - event_sequence=2, - plan=None, - command="audit", + envelope: IntegrationResultEnvelope = written_integration_record( + result=result, terminal=terminal, directory=tmp_path, command="audit" ) - assert envelope is not None decoded: IntegrationResultEnvelope = IntegrationResultEnvelope.from_json(envelope.to_json()) check: IntegrationCheckResult = decoded.checks[0] assert check.passed is test_case.expected_passed diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py index be63cd2796..e89b3c5ddc 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.planner._helpers.resolve.cursor import compute_cursor_bounds @@ -17,7 +18,6 @@ ModelPlanEntry, ) from sqlbuild.compiler.planner.types import BackfillAction, CursorWatermarkMode -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.cursor_algebra.models import IntegerValue from sqlbuild.executor.run._helpers.materializations.microbatch import ( _clamp_intervals_to_model_domain, diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index 2262717a20..18478879af 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -10,10 +10,10 @@ from types import ModuleType from typing import Any +from sqlbuild._native import NativeProgressProjector from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection -from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector from sqlbuild.observability import ( EventDispatcher, LifecycleEvent, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_output_degradation.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_output_degradation.py index cc998f5101..337ce0949e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_output_degradation.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_output_degradation.py @@ -61,9 +61,7 @@ def test_given_completed_execution_when_json_projection_fails_then_exit_status_r ) monkeypatch.setattr(outputs, "format_build_footer", Mock(return_value="completed")) monkeypatch.setattr( - outputs.RunCostStore, - "output_payload", - Mock(return_value={"unserializable": object()}), + outputs, "cost_output_payload", Mock(return_value={"unserializable": object()}) ) outputs.write_build_completion_output( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/helpers.py index 4112022f5d..505faaad65 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/helpers.py @@ -2,7 +2,7 @@ from dataclasses import replace -from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector +from sqlbuild._native import NativeProgressProjector from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.runtime.observability.types import JSONValue diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py index 149e1f5fa0..dc18c8a95e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py @@ -10,10 +10,10 @@ import pytest import sqlbuild.cli.commands._helpers.check.execution as check_execution +from sqlbuild._native import NativeProgressProjector from sqlbuild.cli.commands._helpers.check.core import write_check_results from sqlbuild.cli.commands._helpers.check.execution import execute_check_plan from sqlbuild.cli.commands.models import CheckExecutionPreparation, CheckInvocation -from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.python_nodes.classes.run_state import PythonNodeRunState from sqlbuild.executor.python_nodes.models import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py index edafe77c06..b2d4fc9d97 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import read_cost_run from sqlbuild.cli.commands._helpers.cost.collection import finalize_build_cost -from sqlbuild.cost.classes.run_cost_store import RunCostStore from sqlbuild.cost.models import CostRunRecord, RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from tests.unit.src.sqlbuild.cli.commands._helpers.cost._test_types import ( @@ -136,7 +136,7 @@ def test_given_collection_exception_when_finalizing_then_warning_is_written_with assert record is not None assert record.cost.status == CostStatus.COLLECTION_FAILED assert record.build_status == "success" - assert RunCostStore.read(project_dir=tmp_path, run_id="run-1") == record + assert read_cost_run(project_dir=tmp_path, run_id="run-1") == record assert "Cost telemetry failed during collection (RuntimeError)" in output.getvalue() assert "build result unchanged" in output.getvalue() @@ -194,7 +194,7 @@ def test_given_successful_collection_when_finalizing_then_record_and_default_war ) assert (record is not None) is test_case.expected_record_present - assert RunCostStore.read(project_dir=tmp_path, run_id="run-1") == record + assert read_cost_run(project_dir=tmp_path, run_id="run-1") == record assert "Cost $0.0300 estimated" in output.getvalue() assert "default $3.00/credit" in output.getvalue() assert "Configure cost.usd_per_credit" in output.getvalue() @@ -214,27 +214,18 @@ def test_given_successful_collection_when_finalizing_then_record_and_default_war def test_given_persistence_failure_when_finalizing_then_warning_does_not_escape( test_case: CostCollectionTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: output: StringIO = StringIO() - def fail_write(**kwargs: Any) -> None: - del kwargs - raise OSError("disk full") - - monkeypatch.setattr( - RunCostStore, - "write", - fail_write, - ) + (tmp_path / "target").write_text("not a directory", encoding="utf-8") record: CostRunRecord | None = finalize_build_cost( build_cost_finalization(tmp_path=tmp_path, adapter=_CostAdapter(), output_stream=output) ) assert (record is not None) is test_case.expected_record_present - assert RunCostStore.read(project_dir=tmp_path, run_id="run-1") is None - assert "OSError" in output.getvalue() + assert read_cost_run(project_dir=tmp_path, run_id="run-1") is None + assert "Cost telemetry failed during artifact persistence" in output.getvalue() assert "build result unchanged" in output.getvalue() @@ -261,7 +252,7 @@ def test_given_output_failure_when_finalizing_then_record_is_preserved_without_r ) assert (record is not None) is test_case.expected_record_present - assert RunCostStore.read(project_dir=tmp_path, run_id="run-1") == record + assert read_cost_run(project_dir=tmp_path, run_id="run-1") == record @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_output.py index 9d5b7dbbee..99e0dcefed 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_output.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.cli.commands._helpers.cost.output import format_cost_breakdown +from sqlbuild._native import format_cost_breakdown from sqlbuild.cost.models import CostRunRecord from tests.unit.src.sqlbuild.cli.commands._helpers.cost._test_types import ( CostOutputTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index e9cac889c9..c193fdb396 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -9,9 +9,9 @@ import pytest +from sqlbuild._native import read_cost_run, write_cost_run from sqlbuild.cli.commands._helpers.cost.refresh import refresh_pending_cost_run from sqlbuild.cli.commands.models import CostCommandRequest -from sqlbuild.cost.classes.run_cost_store import RunCostStore from sqlbuild.cost.constants import LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX from sqlbuild.cost.models import CostRunRecord, RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus @@ -79,7 +79,7 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh ) -> None: now: datetime = datetime(2026, 8, 23, tzinfo=UTC) record: CostRunRecord = build_cost_run_record(run_id="run-pending", completed_at=now) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=replace(record, cost=replace(record.cost, status=test_case.initial_status)), ) @@ -119,7 +119,7 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh refresh_pending_cost_run(request) - refreshed: CostRunRecord | None = RunCostStore.read(project_dir=tmp_path, run_id="run-pending") + refreshed: CostRunRecord | None = read_cost_run(project_dir=tmp_path, run_id="run-pending") assert refreshed is not None assert refreshed.cost.status.value == test_case.expected_status assert refreshed.cost.estimated_usd == Decimal("0.03") @@ -175,7 +175,7 @@ def test_given_useful_partial_when_refresh_degrades_then_persisted_summary_is_pr observed_statement_count=17, query_count=2, ) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=replace(record, cost=useful_partial), ) @@ -219,7 +219,7 @@ def test_given_useful_partial_when_refresh_degrades_then_persisted_summary_is_pr refresh_pending_cost_run(request) - refreshed: CostRunRecord | None = RunCostStore.read(project_dir=tmp_path, run_id="run-partial") + refreshed: CostRunRecord | None = read_cost_run(project_dir=tmp_path, run_id="run-partial") assert refreshed is not None assert refreshed.cost.status == test_case.expected_status assert refreshed.cost.observed_statement_count == test_case.expected_observed_statement_count @@ -258,7 +258,7 @@ def test_given_persisted_ledger_failure_when_refresh_has_no_coverage_then_failur limitations=(failure_limitation,), message="Cost telemetry was only partially persisted.", ) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=replace(record, cost=persisted_failure), ) @@ -302,7 +302,7 @@ def test_given_persisted_ledger_failure_when_refresh_has_no_coverage_then_failur refresh_pending_cost_run(request) - refreshed: CostRunRecord | None = RunCostStore.read( + refreshed: CostRunRecord | None = read_cost_run( project_dir=tmp_path, run_id="run-ledger-failure" ) assert refreshed is not None diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/_test_types.py index e05cd444e6..1f9eab986f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/_test_types.py @@ -2,17 +2,6 @@ from dataclasses import dataclass -from sqlbuild.cli.commands.models import DebugResult - - -@dataclass(frozen=True) -class DebugOutputTestCase: - description: str - result: DebugResult - expected_text: str - expected_json_fragment: str - expected_color_fragments: tuple[str, ...] = () - @dataclass(frozen=True) class DebugWarehouseVarsTestCase: diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py index 33ab46d16d..42fbaaa224 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py @@ -2,13 +2,13 @@ from __future__ import annotations +import json from pathlib import Path import pytest -from sqlbuild.cli.commands._helpers.debug.checks import build_debug_result +from sqlbuild._native import run_debug_command from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope -from sqlbuild.cli.commands.models import DebugResult from tests.unit.src.sqlbuild.cli.commands._helpers.debug._test_types import ( DebugWarehouseVarsTestCase, ) @@ -50,16 +50,21 @@ ids=lambda case: case.description, ) def test_given_cli_vars_when_building_debug_result_then_groups_use_invocation_vars( - test_case: DebugWarehouseVarsTestCase, tmp_path: Path + test_case: DebugWarehouseVarsTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: (tmp_path / "sqlbuild_project.toml").write_text(test_case.project_contents, encoding="utf-8") with command_warehouse_scope(command="debug", cli_warehouse=None, cli_vars=test_case.cli_vars): - result: DebugResult = build_debug_result(project_dir=tmp_path, check_connection=False) + exit_code: int = run_debug_command( + project_dir=tmp_path, no_connection=True, json_output=True + ) + payload: dict[str, list[dict[str, str]]] = json.loads(capsys.readouterr().out) + lines: list[dict[str, str]] = payload["configuration"] + payload["connection"] reported: set[tuple[str, str, str]] = { - (line.label, line.message, line.status_message or "") for line in result.lines + (line["label"], line["message"], line.get("status_message", "")) for line in lines } + assert exit_code == 0 assert set(test_case.expected_lines) <= reported diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_output.py deleted file mode 100644 index bf3d62f463..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_output.py +++ /dev/null @@ -1,92 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.debug.output import format_debug_json, format_debug_text -from sqlbuild.cli.commands.models import ( - DebugLine, - DebugResult, -) -from sqlbuild.cli.commands.types import DebugCheckStatus -from tests.unit.src.sqlbuild.cli.commands._helpers.debug._test_types import ( - DebugOutputTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - DebugOutputTestCase( - description="renders diagnostics sections and json sections", - result=DebugResult( - runtime=(DebugLine("sqlbuild version", "0.2.1"),), - configuration=( - DebugLine( - "project file", - "/repo/sqlbuild_project.toml", - DebugCheckStatus.OK, - "found and valid", - ), - ), - providers=( - DebugLine("providers", "1", DebugCheckStatus.OK, "discovered"), - DebugLine( - "marker_provider", - "providers/marker.py:MarkerProvider", - DebugCheckStatus.OK, - "valid settings", - ), - ), - connection=( - DebugLine("token", "****"), - DebugLine( - "connection test", "", DebugCheckStatus.ERROR, "authentication failed" - ), - DebugLine("query test", "", DebugCheckStatus.SKIP, "connection failed"), - ), - ), - expected_text=( - "\n" - "SQLBuild Diagnostics\n" - "\n" - "Runtime:\n" - " sqlbuild version: 0.2.1\n" - "\n" - "Configuration:\n" - " project file: /repo/sqlbuild_project.toml [OK found and valid]\n" - "\n" - "Providers:\n" - " providers: 1 [OK discovered]\n" - " marker_provider: providers/marker.py:MarkerProvider [OK valid settings]\n" - "\n" - "Connection:\n" - " token: ****\n" - " connection test: [ERROR authentication failed]\n" - " query test: [SKIP connection failed]\n" - "\n" - ), - expected_json_fragment='"success": false', - expected_color_fragments=( - "\033[34m\033[1mSQLBuild Diagnostics\033[0m", - "\033[1mRuntime:\033[0m", - "\033[32m[OK found and valid]\033[0m", - "\033[38;5;167m[ERROR authentication failed]\033[0m", - "\033[2m[SKIP connection failed]\033[0m", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_debug_result_when_formatting_then_renders_expected_outputs( - test_case: DebugOutputTestCase, -) -> None: - text: str = format_debug_text(result=test_case.result, use_color=False) - color_text: str = format_debug_text(result=test_case.result, use_color=True) - json_text: str = format_debug_json(test_case.result) - - assert text == test_case.expected_text - assert test_case.expected_json_fragment in json_text - assert '"status": "ERROR"' in json_text - assert '"status_message": "authentication failed"' in json_text - for fragment in test_case.expected_color_fragments: - assert fragment in color_text diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py index af4c9a2860..0a6a73f8e7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py @@ -1,22 +1,11 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import UTC, datetime -from sqlbuild.cli.commands.models import FreshnessCommandResult from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord -@dataclass(frozen=True) -class FreshnessOutputTestCase: - description: str - result: FreshnessCommandResult - expected_text_fragments: tuple[str, ...] - expected_color_fragments: tuple[str, ...] - expected_summary: dict[str, int] - expected_json_age_statuses: dict[str, str | None] - - @dataclass(frozen=True) class FreshnessObservationTestCase: description: str @@ -25,5 +14,5 @@ class FreshnessObservationTestCase: expected_statuses: dict[str, str] expected_versions: dict[str, str] previous_records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] | None = None - expected_age_statuses: dict[str, str] | None = None + expected_age_statuses: dict[str, str] = field(default_factory=dict) observed_at: datetime = datetime(2026, 1, 1, 0, 0, 0, tzinfo=UTC) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/test_freshness_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/test_freshness_output.py deleted file mode 100644 index e58d3287cd..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/test_freshness_output.py +++ /dev/null @@ -1,141 +0,0 @@ -from __future__ import annotations - -import json - -import pytest - -from sqlbuild.cli.commands._helpers.freshness.output import ( - format_freshness_json, - format_freshness_text, -) -from sqlbuild.cli.commands.models import ( - FreshnessCommandResult, - FreshnessSourceResult, -) -from sqlbuild.cli.commands.types import FreshnessSourceStatus -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus -from tests.unit.src.sqlbuild.cli.commands._helpers.freshness._test_types import ( - FreshnessOutputTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - FreshnessOutputTestCase( - description="formats observed unknown and error sources", - result=FreshnessCommandResult( - sources=( - FreshnessSourceResult( - name="raw_orders", - status=FreshnessSourceStatus.OBSERVED, - strategy="timestamp", - value_kind="timestamp", - current_data_version="2026-01-01T00:00:00", - lag_tolerance="15m", - age_status=SourceFreshnessAgeStatus.WARN, - ), - FreshnessSourceResult( - name="raw_clicks", - status=FreshnessSourceStatus.OBSERVED, - strategy="timestamp", - value_kind="timestamp", - current_data_version="2026-01-01T00:30:00", - age_status=SourceFreshnessAgeStatus.PASS, - ), - FreshnessSourceResult( - name="raw_shipments", - status=FreshnessSourceStatus.OBSERVED, - strategy="timestamp", - value_kind="timestamp", - current_data_version="2025-12-31T20:00:00", - age_status=SourceFreshnessAgeStatus.ERROR, - ), - FreshnessSourceResult( - name="raw_inventory", - status=FreshnessSourceStatus.OBSERVED, - strategy="integer", - value_kind="integer", - current_data_version="42", - age_status=SourceFreshnessAgeStatus.UNKNOWN, - ), - FreshnessSourceResult( - name="raw_payments", - status=FreshnessSourceStatus.UNKNOWN, - message="no freshness config and adapter metadata unavailable", - ), - FreshnessSourceResult( - name="raw_events", - status=FreshnessSourceStatus.ERROR, - message="freshness query failed", - ), - ) - ), - expected_text_fragments=( - "Observed (4)", - "├── raw_orders", - "value 2026-01-01T00:00:00 kind timestamp via timestamp tolerance 15m age warn", - "value 2026-01-01T00:30:00 kind timestamp via timestamp age pass", - "value 2025-12-31T20:00:00 kind timestamp via timestamp age error", - "└── raw_inventory", - "value 42 kind integer via integer age unknown", - "Unknown (1)", - "└── raw_payments", - "no freshness config and adapter metadata unavailable", - "Errors (1)", - "└── raw_events", - "freshness query failed", - "OBSERVED=4 CHANGED=0 UNCHANGED=0 TOLERATED=0 UNKNOWN=1 ERROR=1", - "Age policy PASS=1 WARN=1 ERROR=1 UNKNOWN=1", - ), - expected_color_fragments=( - "\033[34m\033[1mSource freshness\033[0m", - "\033[33mage warn\033[0m", - "\033[32mage pass\033[0m", - "\033[38;5;167mage error\033[0m", - "\033[33mno freshness config and adapter metadata unavailable\033[0m", - "\033[38;5;167mfreshness query failed\033[0m", - "\033[2mUNKNOWN=\033[0m\033[33m1\033[0m", - "\033[2mERROR=\033[0m\033[38;5;167m1\033[0m", - ), - expected_summary={ - "observed": 4, - "changed": 0, - "unchanged": 0, - "tolerated": 0, - "unknown": 1, - "errors": 1, - "age_pass": 1, - "age_warn": 1, - "age_error": 1, - "age_unknown": 1, - }, - expected_json_age_statuses={ - "raw_orders": "warn", - "raw_clicks": "pass", - "raw_shipments": "error", - "raw_inventory": "unknown", - "raw_payments": None, - "raw_events": None, - }, - ) - ], - ids=lambda case: case.description, -) -def test_given_freshness_result_when_formatting_then_includes_status_groups( - test_case: FreshnessOutputTestCase, -) -> None: - text_output: str = format_freshness_text(result=test_case.result) - color_text_output: str = format_freshness_text(result=test_case.result, use_color=True) - json_output: dict[str, object] = json.loads(format_freshness_json(test_case.result)) - - fragment: str - for fragment in test_case.expected_text_fragments: - assert fragment in text_output - for fragment in test_case.expected_color_fragments: - assert fragment in color_text_output - assert json_output["summary"] == test_case.expected_summary - sources: list[dict[str, object]] = json_output["sources"] - assert {str(source["name"]): source["age_status"] for source in sources} == ( - test_case.expected_json_age_statuses - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/test_observe.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/test_observe.py index 8145a197e4..737e09fa6b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/test_observe.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/test_observe.py @@ -1,15 +1,11 @@ from __future__ import annotations -from collections import defaultdict -from typing import Any, cast +import json +from typing import Any import pytest -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.cli.commands._helpers.freshness.observe import ( - observe_source_freshness_for_command, -) -from sqlbuild.cli.commands.models import FreshnessCommandResult +from sqlbuild._native import observe_source_freshness_json from tests.unit.src.sqlbuild.cli.commands._helpers.freshness._test_types import ( FreshnessObservationTestCase, ) @@ -114,35 +110,31 @@ def test_given_sources_when_observing_freshness_then_classifies_sources( ) -> None: adapter: FreshnessRecordingAdapter = FreshnessRecordingAdapter() - result: FreshnessCommandResult = observe_source_freshness_for_command( - adapter=cast(StrictAdapter, adapter), - connection=object(), - sources=freshness_sources(), - select=test_case.select, - exclude=test_case.exclude, - observed_at=test_case.observed_at, - previous_records=test_case.previous_records, + payload: dict[str, Any] = json.loads( + observe_source_freshness_json( + adapter=adapter, + connection=object(), + sources=freshness_sources(), + select=test_case.select, + exclude=test_case.exclude, + observed_at=test_case.observed_at, + previous_records=test_case.previous_records, + ) ) + sources: list[dict[str, Any]] = payload["sources"] - statuses: dict[str, str] = {source.name: source.status for source in result.sources} + statuses: dict[str, str] = {source["name"]: source["status"] for source in sources} versions: dict[str, str] = { - source.name: source.current_data_version or "" for source in result.sources + source["name"]: source["current_data_version"] or "" for source in sources } assert statuses == test_case.expected_statuses assert { name: versions[name] for name in test_case.expected_versions } == test_case.expected_versions - sources_by_age_status_presence: defaultdict[bool, list[str]] = defaultdict(list) - for source in result.sources: - sources_by_age_status_presence[source.age_status is not None].append(source.name) - expected_age_statuses: dict[str, str] = test_case.expected_age_statuses or {} - assert set(sources_by_age_status_presence[True]) == set(expected_age_statuses) - sources_by_name: dict[str, Any] = {source.name: source for source in result.sources} - assert len(sources_by_name) == len(result.sources) - for name, expected_age_status in expected_age_statuses.items(): - age_status: Any = sources_by_name[name].age_status - assert age_status is not None - assert age_status.value == expected_age_status + age_statuses: dict[str, str | None] = { + source["name"]: source["age_status"] for source in sources + } + assert age_statuses == dict.fromkeys(statuses) | test_case.expected_age_statuses @pytest.mark.parametrize( @@ -163,19 +155,22 @@ def test_given_adapter_metadata_source_when_observing_freshness_then_uses_table_ ) -> None: adapter: FreshnessRecordingAdapter = FreshnessRecordingAdapter(table_metadata_supported=True) - result: FreshnessCommandResult = observe_source_freshness_for_command( - adapter=cast(StrictAdapter, adapter), - connection=object(), - sources=adapter_metadata_sources(), - select=test_case.select, - exclude=test_case.exclude, - observed_at=test_case.observed_at, - previous_records=test_case.previous_records, + payload: dict[str, Any] = json.loads( + observe_source_freshness_json( + adapter=adapter, + connection=object(), + sources=adapter_metadata_sources(), + select=test_case.select, + exclude=test_case.exclude, + observed_at=test_case.observed_at, + previous_records=test_case.previous_records, + ) ) + sources: list[dict[str, Any]] = payload["sources"] - statuses: dict[str, str] = {source.name: source.status for source in result.sources} + statuses: dict[str, str] = {source["name"]: source["status"] for source in sources} versions: dict[str, str] = { - source.name: source.current_data_version or "" for source in result.sources + source["name"]: source["current_data_version"] or "" for source in sources } assert statuses == test_case.expected_statuses assert versions == test_case.expected_versions diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py index cb73832d38..16012e165a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py @@ -2,13 +2,9 @@ from __future__ import annotations -from types import SimpleNamespace -from typing import cast - import pytest -from sqlbuild.cli.commands._helpers.janitor_output.outputs import write_janitor_completion -from sqlbuild.cli.commands.models import JanitorInvocation +from sqlbuild._native import render_janitor_completion from sqlbuild.executor.janitor.models import JanitorExecutionResult from tests.unit.src.sqlbuild.cli.commands._helpers.janitor_output._test_types import ( JanitorCompletionTestCase, @@ -48,7 +44,7 @@ ids=lambda case: case.description, ) def test_given_janitor_counts_when_writing_completion_then_nouns_match_counts( - test_case: JanitorCompletionTestCase, capsys: pytest.CaptureFixture[str] + test_case: JanitorCompletionTestCase, ) -> None: result: JanitorExecutionResult = JanitorExecutionResult( archived=placeholder_candidates(test_case.archived), @@ -56,11 +52,9 @@ def test_given_janitor_counts_when_writing_completion_then_nouns_match_counts( pruned_direct_state=placeholder_candidates(test_case.pruned_direct_state), ) - write_janitor_completion( - invocation=cast(JanitorInvocation, SimpleNamespace(use_color=False)), result=result - ) + output: str = render_janitor_completion(result=result, use_color=False) - assert capsys.readouterr().out == test_case.expected_output + assert output == test_case.expected_output if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/_test_types.py index 60360ccf56..25f0494986 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/_test_types.py @@ -3,70 +3,6 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class LineageSelectionTestCase: - description: str - target: str - direction: str - depth: int | None - expected_node_ids: tuple[str, ...] - expected_edge_ids: tuple[str, ...] - - -@dataclass(frozen=True) -class ColumnLineageSelectionTestCase: - description: str - target: str - direction: str - depth: int | None - expected_resource_name: str - expected_column_name: str - expected_trace_ids: tuple[str, ...] - expected_analyzed_model_names: tuple[str, ...] - expected_truncated: bool - - -@dataclass(frozen=True) -class LineageSelectorDepthErrorTestCase: - description: str - select: tuple[str, ...] - expected_error_fragment: str - - -@dataclass(frozen=True) -class LineageOutputTestCase: - description: str - output_format: str - expected_output: str - expected_color_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class ColumnLineageOutputTestCase: - description: str - output_format: str - expected_output: str - expected_color_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class LargeColumnLineageOutputTestCase: - description: str - expected_included_fragment: str - expected_excluded_fragment: str - expected_summary_fragment: str - expected_json_tip_fragment: str - - -@dataclass(frozen=True) -class LineageCompiledGraphRequirementTestCase: - description: str - targets: tuple[str, ...] - select: tuple[str, ...] - include_uses: bool - expected_compiled_graph_required: bool - - @dataclass(frozen=True) class LineageFingerprintEnvironmentTestCase: description: str @@ -85,57 +21,6 @@ class LineageFingerprintAvailabilityTestCase: expected_available: bool -@dataclass(frozen=True) -class LineagePathSelectorTestCase: - description: str - select: tuple[str, ...] - expected_node_ids: tuple[str, ...] - - -@dataclass(frozen=True) -class LineagePathSelectorErrorTestCase: - description: str - select: tuple[str, ...] - expected_error_fragment: str - - -@dataclass(frozen=True) -class SharedSelectorParityTestCase: - description: str - select: tuple[str, ...] - exclude: tuple[str, ...] - expected_node_ids: tuple[str, ...] - - -@dataclass(frozen=True) -class SharedSelectorErrorParityTestCase: - description: str - select: tuple[str, ...] - expected_code: str - - -@dataclass(frozen=True) -class NormalizeLineageTargetTestCase: - description: str - target: str - expected_target: str - - -@dataclass(frozen=True) -class NormalizeLineageTargetErrorTestCase: - description: str - target: str - expected_code: str - - -@dataclass(frozen=True) -class SharedSelectorDepthTestCase: - description: str - select: tuple[str, ...] - direction: str | None - expected_node_ids: tuple[str, ...] - - @dataclass(frozen=True) class FingerprintedFilesTestCase: """Authored files whose relation lineage fingerprint hashes exactly the expected inputs.""" @@ -145,7 +30,7 @@ class FingerprintedFilesTestCase: links: dict[str, str] unreadable_directories: tuple[str, ...] environment: dict[str, str] - cli_vars: dict[str, object] | None + cli_vars: dict[str, object] expected_hashed_files: tuple[str, ...] expected_environment_names: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/helpers.py index 039fc21d05..dda2d00371 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/helpers.py @@ -1,262 +1,35 @@ -"""Helpers for lineage helper tests.""" +"""Helpers for lineage cache fingerprint tests.""" from __future__ import annotations import glob import hashlib import inspect +import json import os -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Callable, Iterator +from importlib.metadata import version from pathlib import Path from types import TracebackType from typing import Protocol import pytest -from sqlbuild.cli.commands.models import ColumnLineageTrace, LineageNode -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSeed, - CompiledSource, - CompileModelConfig, - InferredColumn, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSchemaFile, - DiscoveredSeedFile, - DiscoveredSourceFile, -) -from sqlbuild.compiler.lineage.models import ColumnLineageEdge, QualifiedLineageColumn -from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, ColumnTransformKind -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.spec.contracts.models import SchemaSeedEntry, SettingsConfig, SourceEntry -from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes +_CACHE_ALGORITHM_VERSION: str = "relation-lineage-graph-v4" -def build_lineage_test_graph() -> ProjectGraph: - """Build a small static graph for lineage tests.""" +def fingerprint_prefix(*, cli_vars: dict[str, object]) -> bytes: + """The cache key prefix: algorithm, installed version and canonical `cli_vars` JSON.""" - raw_orders_key: CompiledObjectKey = CompiledObjectKey(CompiledResourceType.SOURCE, "raw_orders") - stg_orders_key: CompiledObjectKey = CompiledObjectKey(CompiledResourceType.MODEL, "stg_orders") - fact_orders_key: CompiledObjectKey = CompiledObjectKey( - CompiledResourceType.MODEL, "fact_orders" - ) - daily_rollup_key: CompiledObjectKey = CompiledObjectKey( - CompiledResourceType.MODEL, "daily_rollup" - ) - waffle_types_key: CompiledObjectKey = CompiledObjectKey( - CompiledResourceType.SEED, "waffle_types" - ) - - schema_file: DiscoveredSchemaFile = DiscoveredSchemaFile( - file_path=Path("schema.yml"), - relative_path=Path("schema.yml"), - contents="", - model_entries=(), - seed_entries=(), - ) - project: CompiledProjectView = CompiledProject( - run_id="test-run", - effective_target_name=None, - effective_connection={}, - effective_vars={}, - settings=SettingsConfig(), - models=( - _model( - "stg_orders", - stg_orders_key, - (raw_orders_key,), - "models/stg_orders.sql", - query_sql="SELECT 1 AS order_id", - inferred_columns=("order_id",), - ), - _model( - "fact_orders", - fact_orders_key, - (stg_orders_key, waffle_types_key), - "models/fact_orders.sql", - query_sql='SELECT order_id FROM __ref("stg_orders")', - inferred_columns=("order_id",), - ), - _model( - "daily_rollup", - daily_rollup_key, - (fact_orders_key,), - "models/daily_rollup.sql", - ), - ), - sources=( - CompiledSource( - key=raw_orders_key, - deps=(), - name="raw_orders", - source_entry=SourceEntry(name="raw_orders", table="raw_orders"), - source_file=DiscoveredSourceFile( - file_path=Path("sources/raw.yml"), - relative_path=Path("sources/raw.yml"), - contents="", - source_entries=(), - ), - ), - ), - seeds=( - CompiledSeed( - key=waffle_types_key, - deps=(), - name="waffle_types", - seed_file=DiscoveredSeedFile( - file_path=Path("seeds/waffle_types.csv"), - relative_path=Path("seeds/waffle_types.csv"), - ), - schema_entry=SchemaSeedEntry(name="waffle_types"), - schema_file=schema_file, - destination=_target("waffle_types"), - ), - ), - ) - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = { - raw_orders_key: (), - waffle_types_key: (), - stg_orders_key: (raw_orders_key,), - fact_orders_key: (stg_orders_key, waffle_types_key), - daily_rollup_key: (fact_orders_key,), - } - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = { - raw_orders_key: (stg_orders_key,), - waffle_types_key: (fact_orders_key,), - stg_orders_key: (fact_orders_key,), - fact_orders_key: (daily_rollup_key,), - daily_rollup_key: (), - } - return project_graph_from_indexes( - project=project, - upstream_deps=upstream_deps, - downstream_deps=downstream_deps, - tag_index={"marts": frozenset({fact_orders_key, daily_rollup_key})}, - path_index={ - stg_orders_key: "staging", - fact_orders_key: "marts", - daily_rollup_key: "marts", - }, - all_keys={ - "raw_orders": raw_orders_key, - "waffle_types": waffle_types_key, - "stg_orders": stg_orders_key, - "fact_orders": fact_orders_key, - "daily_rollup": daily_rollup_key, - }, - ) - - -def node_ids(graph_nodes: Iterable[LineageNode]) -> tuple[str, ...]: - return tuple(f"{node.key.resource_type}:{node.key.name}" for node in graph_nodes) - - -def edge_ids( - graph_edges: Iterable[tuple[CompiledObjectKey, CompiledObjectKey]], -) -> tuple[str, ...]: - return tuple( - f"{parent.resource_type}:{parent.name}->{child.resource_type}:{child.name}" - for parent, child in graph_edges - ) - - -def build_column_lineage_trace() -> ColumnLineageTrace: - target: QualifiedLineageColumn = QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name="fact_orders", - column_name="line_total_cents", - ) - return ColumnLineageTrace( - target=target, - direction="upstream", - max_depth=3, - analyzed_model_count=7, - truncated=True, - trace=( - ColumnLineageEdge( - source=QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name="stg_orders", - column_name="quantity", - ), - target=target, - transform_kind=ColumnTransformKind.EXPRESSION, - confidence=ColumnLineageConfidence.HIGH, - ), - ColumnLineageEdge( - source=QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name="wide_model_1", - column_name="line_total_cents", - ), - target=target, - transform_kind=ColumnTransformKind.STAR, - confidence=ColumnLineageConfidence.MEDIUM, - ), - ), - ) - - -def build_large_column_lineage_trace() -> ColumnLineageTrace: - target: QualifiedLineageColumn = QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name="fact_orders", - column_name="order_id", - ) - return ColumnLineageTrace( - target=target, - direction="downstream", - trace=tuple( - ColumnLineageEdge( - source=target, - target=QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name=f"consumer_{index:02d}", - column_name="order_id", - ), - transform_kind=ColumnTransformKind.DIRECT, - confidence=ColumnLineageConfidence.HIGH, - ) - for index in range(30) - ), - ) - - -def _model( - name: str, - key: CompiledObjectKey, - deps: tuple[CompiledObjectKey, ...], - relative_path: str, - query_sql: str = "SELECT 1", - inferred_columns: tuple[str, ...] = (), -) -> CompiledModelView: - return CompiledModel( - key=key, - deps=deps, - name=name, - relative_path=Path(relative_path), - query_sql=query_sql, - config=CompileModelConfig(), - destination=_target(name), - inferred_columns=tuple(InferredColumn(column) for column in inferred_columns), - ) - - -def _target(name: str) -> CompiledRelationLocation: - return CompiledRelationLocation( - database=None, - schema="main", - name=name, - qualified_name=f"main.{name}", + return ( + _CACHE_ALGORITHM_VERSION.encode("ascii") + + version("sqlbuild").encode("utf-8") + + json.dumps( + cli_vars, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py index 1e54dbf9c9..1b7054ac96 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py @@ -6,11 +6,7 @@ import pytest -from sqlbuild.cli.commands._helpers.lineage.cache import ( - _INTERRUPTED_LISTING_UNCACHEABLE, - _fingerprint_prefix, - relation_lineage_fingerprint, -) +from sqlbuild._native import lineage_cache_fingerprint from tests.unit.src.sqlbuild.cli.commands._helpers.lineage._test_types import ( FingerprintedFilesTestCase, InterruptedListingFingerprintTestCase, @@ -20,6 +16,7 @@ ) from tests.unit.src.sqlbuild.cli.commands._helpers.lineage.helpers import ( expected_fingerprint, + fingerprint_prefix, interrupt_listing, ) @@ -93,10 +90,10 @@ def test_given_referenced_environment_change_when_fingerprinting_then_invalidate ) -> None: (tmp_path / "sqlbuild_project.toml").write_text(test_case.config, encoding="utf-8") monkeypatch.setenv(test_case.environment_name, test_case.first_value) - first: str | None = relation_lineage_fingerprint(project_dir=tmp_path, cli_vars=None) + first: str | None = lineage_cache_fingerprint(project_dir=tmp_path, cli_vars=None) monkeypatch.setenv(test_case.environment_name, test_case.second_value) - second: str | None = relation_lineage_fingerprint(project_dir=tmp_path, cli_vars=None) + second: str | None = lineage_cache_fingerprint(project_dir=tmp_path, cli_vars=None) assert (first == second) is test_case.expected_equal @@ -141,7 +138,7 @@ def test_given_dynamic_invocation_context_when_fingerprinting_then_disables_cach authored_path.parent.mkdir(parents=True, exist_ok=True) authored_path.write_text(test_case.config, encoding="utf-8") - observed: str | None = relation_lineage_fingerprint(project_dir=tmp_path, cli_vars=None) + observed: str | None = lineage_cache_fingerprint(project_dir=tmp_path, cli_vars=None) assert (observed is not None) is test_case.expected_available @@ -165,7 +162,7 @@ def test_given_dynamic_invocation_context_when_fingerprinting_then_disables_cach links={}, unreadable_directories=("models/locked",), environment={"ORDERS_SCHEMA": "orders_dev"}, - cli_vars=None, + cli_vars={}, expected_hashed_files=_LAYOUT_UNLINKED_HASHED_FILES + (("models/locked/hidden.sql",) if sys.platform == "win32" else ()), expected_environment_names=_LAYOUT_ENVIRONMENT_NAMES, @@ -180,7 +177,7 @@ def test_given_dynamic_invocation_context_when_fingerprinting_then_disables_cach links={}, unreadable_directories=(), environment={"ORDERS_SCHEMA": "orders_dev"}, - cli_vars=None, + cli_vars={}, expected_hashed_files=_LAYOUT_UNLINKED_HASHED_FILES, expected_environment_names=_LAYOUT_ENVIRONMENT_NAMES, ), @@ -190,7 +187,7 @@ def test_given_dynamic_invocation_context_when_fingerprinting_then_disables_cach links={}, unreadable_directories=(), environment={"ORDERS_SCHEMA": "orders_dev"}, - cli_vars=None, + cli_vars={}, expected_hashed_files=_LAYOUT_UNLINKED_HASHED_FILES, expected_environment_names=_LAYOUT_ENVIRONMENT_NAMES, ), @@ -214,7 +211,7 @@ def test_given_authored_files_when_fingerprinting_then_hashes_exactly_the_expect monkeypatch.setenv(name, value) monkeypatch.delenv("ORDERS_REGION", raising=False) - observed: str | None = relation_lineage_fingerprint( + observed: str | None = lineage_cache_fingerprint( project_dir=tmp_path, cli_vars=test_case.cli_vars ) @@ -222,7 +219,7 @@ def test_given_authored_files_when_fingerprinting_then_hashes_exactly_the_expect (tmp_path / relative_path).chmod(0o755) assert observed == expected_fingerprint( project_dir=tmp_path, - prefix=_fingerprint_prefix(cli_vars=test_case.cli_vars), + prefix=fingerprint_prefix(cli_vars=test_case.cli_vars), hashed_files=test_case.expected_hashed_files, environment_names=test_case.expected_environment_names, ) @@ -272,7 +269,7 @@ def test_given_unhashable_inputs_when_fingerprinting_then_the_cache_is_disabled( path.parent.mkdir(parents=True, exist_ok=True) _ = path.write_text(contents, encoding="utf-8") - observed: str | None = relation_lineage_fingerprint(project_dir=tmp_path, cli_vars=None) + observed: str | None = lineage_cache_fingerprint(project_dir=tmp_path, cli_vars=None) assert observed is test_case.expected_fingerprint @@ -305,4 +302,3 @@ def test_given_interrupted_listing_when_python_globs_then_native_policy_matches_ listed = list(tmp_path.rglob("*")) assert (not listed) is (sys.version_info < test_case.expected_uncacheable_before) - assert (not listed) is _INTERRUPTED_LISTING_UNCACHEABLE diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_execution.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_execution.py deleted file mode 100644 index 26342496bd..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_execution.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.lineage.execution import _requires_compiled_graph -from sqlbuild.cli.commands.models import LineageCommandRequest -from tests.unit.src.sqlbuild.cli.commands._helpers.lineage._test_types import ( - LineageCompiledGraphRequirementTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - LineageCompiledGraphRequirementTestCase( - description="relation target uses structural lineage", - targets=("fact_orders",), - select=(), - include_uses=False, - expected_compiled_graph_required=False, - ), - LineageCompiledGraphRequirementTestCase( - description="selector uses structural lineage", - targets=(), - select=("fact_orders+",), - include_uses=False, - expected_compiled_graph_required=False, - ), - LineageCompiledGraphRequirementTestCase( - description="column target retains column analysis", - targets=("fact_orders.order_id",), - select=(), - include_uses=False, - expected_compiled_graph_required=True, - ), - LineageCompiledGraphRequirementTestCase( - description="semantic uses retain the compiled graph", - targets=("fact_orders",), - select=(), - include_uses=True, - expected_compiled_graph_required=True, - ), - ), - ids=lambda case: case.description, -) -def test_given_lineage_request_when_checking_analysis_requirement_then_matches_target_granularity( - test_case: LineageCompiledGraphRequirementTestCase, -) -> None: - request: LineageCommandRequest = LineageCommandRequest( - project_dir=None, - targets=test_case.targets, - select=test_case.select, - include_uses=test_case.include_uses, - ) - - observed: bool = _requires_compiled_graph(request=request) - - assert observed is test_case.expected_compiled_graph_required diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_output.py deleted file mode 100644 index b983cc5fca..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_output.py +++ /dev/null @@ -1,248 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable - -import pytest - -from sqlbuild.cli.commands._helpers.lineage.output import ( - format_column_lineage_json, - format_column_lineage_list, - format_column_lineage_tree, - format_lineage_list, - format_lineage_tree, -) -from sqlbuild.cli.commands._helpers.lineage.selection import select_target_lineage -from sqlbuild.cli.commands.models import ColumnLineageTrace, LineageGraph -from sqlbuild.compiler.pipeline.models import ProjectGraph -from tests.unit.src.sqlbuild.cli.commands._helpers.lineage._test_types import ( - ColumnLineageOutputTestCase, - LargeColumnLineageOutputTestCase, - LineageOutputTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.lineage.helpers import ( - build_column_lineage_trace, - build_large_column_lineage_trace, - build_lineage_test_graph, -) - - -@pytest.mark.parametrize( - "test_case", - [ - LineageOutputTestCase( - description="renders tree output with branch glyphs and metadata", - output_format="tree", - expected_output=( - "Lineage model fact_orders models/fact_orders.sql upstream\n" - "├── model stg_orders models/stg_orders.sql\n" - "│ └── source raw_orders sources/raw.yml\n" - "└── seed waffle_types seeds/waffle_types.csv" - ), - expected_color_fragments=( - "Lineage", - "\033[2mmodel\033[0m \033[1mfact_orders\033[0m", - "\033[2mupstream\033[0m", - "\033[2m├── \033[0m\033[2mmodel\033[0m \033[1mstg_orders\033[0m", - ), - ), - LineageOutputTestCase( - description="renders list output as aligned edge list", - output_format="list", - expected_output=( - "model:stg_orders -> model:fact_orders\n" - "seed:waffle_types -> model:fact_orders\n" - "source:raw_orders -> model:stg_orders" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_lineage_graph_when_formatting_then_returns_expected_human_output( - test_case: LineageOutputTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - lineage_graph: LineageGraph = select_target_lineage( - graph=graph, - targets=("fact_orders",), - direction="upstream", - depth=None, - ) - - renderers: dict[str, Callable[[LineageGraph], str]] = { - "tree": lambda graph: format_lineage_tree(graph=graph, use_color=False), - "list": lambda graph: format_lineage_list(graph=graph, use_color=False), - } - result: str = renderers[test_case.output_format](lineage_graph) - - assert result == test_case.expected_output - - -@pytest.mark.parametrize( - "test_case", - [ - ColumnLineageOutputTestCase( - description="renders column tree output with end-user dependency labels", - output_format="tree", - expected_output=( - "Column trace fact_orders.line_total_cents upstream\n\n" - "├── stg_orders.quantity (expression)\n" - "└── wide_model_1.line_total_cents (from SELECT *)" - ), - expected_color_fragments=( - "Column trace", - "\033[1mfact_orders.line_total_cents\033[0m", - "\033[2mupstream\033[0m", - "\033[2m├── \033[0m\033[1mstg_orders.quantity\033[0m", - ), - ), - ColumnLineageOutputTestCase( - description="renders column list output as dependency list", - output_format="list", - expected_output=( - "Column dependencies\n\n" - "stg_orders.quantity -> fact_orders.line_total_cents expression\n" - "wide_model_1.line_total_cents -> fact_orders.line_total_cents from SELECT *" - ), - ), - ColumnLineageOutputTestCase( - description="renders column json output with graph terms for machines", - output_format="json", - expected_output=( - '{\n "target": {\n "resource_type": "model",\n' - ' "resource_name": "fact_orders",\n' - ' "column_name": "line_total_cents"\n },\n' - ' "direction": "upstream",\n "metadata": {\n' - ' "mode": "rich",\n "max_depth": 3,\n' - ' "analyzed_models": 7,\n "truncated": true\n },\n' - ' "trace": [\n {\n' - ' "source": {\n "resource_type": "model",\n' - ' "resource_name": "stg_orders",\n' - ' "column_name": "quantity"\n },\n' - ' "target": {\n "resource_type": "model",\n' - ' "resource_name": "fact_orders",\n' - ' "column_name": "line_total_cents"\n },\n' - ' "transform": "expression",\n' - ' "confidence": "high"\n },\n {\n' - ' "source": {\n "resource_type": "model",\n' - ' "resource_name": "wide_model_1",\n' - ' "column_name": "line_total_cents"\n },\n' - ' "target": {\n "resource_type": "model",\n' - ' "resource_name": "fact_orders",\n' - ' "column_name": "line_total_cents"\n },\n' - ' "transform": "star",\n' - ' "confidence": "medium"\n }\n ]\n}' - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_column_lineage_trace_when_formatting_then_returns_expected_output( - test_case: ColumnLineageOutputTestCase, -) -> None: - trace: ColumnLineageTrace = build_column_lineage_trace() - renderers: dict[str, Callable[[ColumnLineageTrace], str]] = { - "tree": lambda trace: format_column_lineage_tree(trace=trace, use_color=False), - "list": lambda trace: format_column_lineage_list(trace=trace, use_color=False), - "json": format_column_lineage_json, - } - - result: str = renderers[test_case.output_format](trace) - - assert result == test_case.expected_output - - -@pytest.mark.parametrize( - "test_case", - [ - LineageOutputTestCase( - description="styles lineage tree semantic parts", - output_format="tree", - expected_output="", - expected_color_fragments=( - "Lineage", - "\033[2mmodel\033[0m \033[1mfact_orders\033[0m", - "\033[2mupstream\033[0m", - "\033[2m├── \033[0m\033[2mmodel\033[0m \033[1mstg_orders\033[0m", - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_lineage_graph_when_formatting_with_color_then_styles_semantic_parts( - test_case: LineageOutputTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - lineage_graph: LineageGraph = select_target_lineage( - graph=graph, - targets=("fact_orders",), - direction="upstream", - depth=None, - ) - color_renderers: dict[str, Callable[[LineageGraph], str]] = { - "tree": lambda graph: format_lineage_tree(graph=graph, use_color=True), - "list": lambda graph: format_lineage_list(graph=graph, use_color=True), - } - - color_result: str = color_renderers[test_case.output_format](lineage_graph) - - for fragment in test_case.expected_color_fragments: - assert fragment in color_result - - -@pytest.mark.parametrize( - "test_case", - [ - ColumnLineageOutputTestCase( - description="styles column lineage tree semantic parts", - output_format="tree", - expected_output="", - expected_color_fragments=( - "Column trace", - "\033[1mfact_orders.line_total_cents\033[0m", - "\033[2mupstream\033[0m", - "\033[2m├── \033[0m\033[1mstg_orders.quantity\033[0m", - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_column_lineage_trace_when_formatting_with_color_then_styles_semantic_parts( - test_case: ColumnLineageOutputTestCase, -) -> None: - trace: ColumnLineageTrace = build_column_lineage_trace() - color_renderers: dict[str, Callable[[ColumnLineageTrace], str]] = { - "tree": lambda trace: format_column_lineage_tree(trace=trace, use_color=True), - "list": lambda trace: format_column_lineage_list(trace=trace, use_color=True), - "json": format_column_lineage_json, - } - - color_result: str = color_renderers[test_case.output_format](trace) - - for fragment in test_case.expected_color_fragments: - assert fragment in color_result - - -@pytest.mark.parametrize( - "test_case", - [ - LargeColumnLineageOutputTestCase( - description="limits large human column traces", - expected_included_fragment="consumer_24.order_id", - expected_excluded_fragment="consumer_25.order_id", - expected_summary_fragment="Showing 25 of 30 columns.", - expected_json_tip_fragment="Use --format json for the full trace.", - ) - ], - ids=lambda case: case.description, -) -def test_given_large_column_trace_when_formatting_tree_then_limits_human_output( - test_case: LargeColumnLineageOutputTestCase, -) -> None: - trace: ColumnLineageTrace = build_large_column_lineage_trace() - - result: str = format_column_lineage_tree(trace=trace, use_color=False) - - assert test_case.expected_included_fragment in result - assert test_case.expected_excluded_fragment not in result - assert test_case.expected_summary_fragment in result - assert test_case.expected_json_tip_fragment in result diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py deleted file mode 100644 index 5094b0e206..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_selection.py +++ /dev/null @@ -1,618 +0,0 @@ -from __future__ import annotations - -from typing import cast - -import pytest - -import sqlbuild._native as native_module -from sqlbuild._native import ProjectColumnLineage -from sqlbuild.cli.commands._helpers.lineage.selection import ( - normalize_lineage_target, - select_column_target_lineage, - select_selector_lineage, - select_target_lineage, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ColumnLineageTrace, LineageGraph -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.models import ( - ColumnLineageEdge, - QualifiedLineageColumn, -) -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors -from tests.unit.src.sqlbuild.cli.commands._helpers.lineage._test_types import ( - ColumnLineageSelectionTestCase, - LineagePathSelectorErrorTestCase, - LineagePathSelectorTestCase, - LineageSelectionTestCase, - LineageSelectorDepthErrorTestCase, - NormalizeLineageTargetErrorTestCase, - NormalizeLineageTargetTestCase, - SharedSelectorDepthTestCase, - SharedSelectorErrorParityTestCase, - SharedSelectorParityTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.lineage.helpers import ( - build_lineage_test_graph, - edge_ids, - node_ids, -) - - -@pytest.mark.parametrize( - "test_case", - [ - LineageSelectionTestCase( - description="limits downstream target lineage to one hop", - target="fact_orders", - direction="downstream", - depth=1, - expected_node_ids=("model:daily_rollup", "model:fact_orders"), - expected_edge_ids=("model:fact_orders->model:daily_rollup",), - ), - LineageSelectionTestCase( - description="keeps full upstream target lineage", - target="fact_orders", - direction="upstream", - depth=None, - expected_node_ids=( - "model:fact_orders", - "model:stg_orders", - "seed:waffle_types", - "source:raw_orders", - ), - expected_edge_ids=( - "model:stg_orders->model:fact_orders", - "seed:waffle_types->model:fact_orders", - "source:raw_orders->model:stg_orders", - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_target_lineage_request_when_selecting_then_returns_expected_subgraph( - test_case: LineageSelectionTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - - result: LineageGraph = select_target_lineage( - graph=graph, - targets=(test_case.target,), - direction=test_case.direction, - depth=test_case.depth, - ) - - assert node_ids(result.nodes) == test_case.expected_node_ids - assert edge_ids(result.edges) == test_case.expected_edge_ids - - -@pytest.mark.parametrize( - "test_case", - [ - ColumnLineageSelectionTestCase( - description="selects upstream column trace for dot target", - target="fact_orders.order_id", - direction="upstream", - depth=None, - expected_resource_name="fact_orders", - expected_column_name="order_id", - expected_trace_ids=("stg_orders.order_id->fact_orders.order_id",), - expected_analyzed_model_names=("fact_orders", "stg_orders"), - expected_truncated=False, - ), - ColumnLineageSelectionTestCase( - description="respects zero depth for column target", - target="fact_orders.order_id", - direction="upstream", - depth=0, - expected_resource_name="fact_orders", - expected_column_name="order_id", - expected_trace_ids=(), - expected_analyzed_model_names=("fact_orders",), - expected_truncated=True, - ), - ColumnLineageSelectionTestCase( - description="selects downstream candidate models for column target", - target="fact_orders.order_id", - direction="downstream", - depth=1, - expected_resource_name="fact_orders", - expected_column_name="order_id", - expected_trace_ids=("fact_orders.order_id->daily_rollup.order_id",), - expected_analyzed_model_names=("daily_rollup", "fact_orders"), - expected_truncated=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_column_target_when_selecting_then_returns_column_trace( - test_case: ColumnLineageSelectionTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - received_model_names: list[frozenset[str]] = [] - received_modes: list[ColumnLineageMode] = [] - - def build_column_lineage_spy( - project: object, dialect: object, mode: object, model_names: object - ) -> ProjectColumnLineage: - del project, dialect - received_model_names.append(cast(frozenset[str], model_names)) - received_modes.append(cast(ColumnLineageMode, mode)) - return ProjectColumnLineage( - models={}, - edges=( - ColumnLineageEdge( - source=QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name="stg_orders", - column_name="order_id", - ), - target=QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name="fact_orders", - column_name="order_id", - ), - ), - ColumnLineageEdge( - source=QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name="fact_orders", - column_name="order_id", - ), - target=QualifiedLineageColumn( - resource_type=CompiledResourceType.MODEL, - resource_name="daily_rollup", - column_name="order_id", - ), - ), - ), - ) - - monkeypatch.setattr(native_module, "build_project_column_lineage", build_column_lineage_spy) - - result: ColumnLineageTrace | None = select_column_target_lineage( - graph=graph, - target=test_case.target, - direction=test_case.direction, - depth=test_case.depth, - mode=ColumnLineageMode.FAST, - ) - - assert result is not None - assert result.target.resource_name == test_case.expected_resource_name - assert result.target.column_name == test_case.expected_column_name - assert ( - tuple( - f"{edge.source.resource_name}.{edge.source.column_name}->{edge.target.resource_name}.{edge.target.column_name}" - for edge in result.trace - ) - == test_case.expected_trace_ids - ) - assert tuple(sorted(received_model_names[0])) == test_case.expected_analyzed_model_names - assert received_modes == [ColumnLineageMode.FAST] - assert result.mode == ColumnLineageMode.FAST - assert result.max_depth == test_case.depth - assert result.analyzed_model_count == len(test_case.expected_analyzed_model_names) - assert result.truncated is test_case.expected_truncated - - -@pytest.mark.parametrize( - "test_case", - [ - LineageSelectionTestCase( - description="trims expanded name selector to requested depth", - target="unused", - direction="unused", - depth=1, - expected_node_ids=("model:fact_orders", "model:stg_orders", "seed:waffle_types"), - expected_edge_ids=( - "model:stg_orders->model:fact_orders", - "seed:waffle_types->model:fact_orders", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_expanded_selector_with_depth_when_selecting_then_trims_selected_subgraph( - test_case: LineageSelectionTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - - result: LineageGraph = select_selector_lineage( - graph=graph, - select=("+fact_orders",), - exclude=(), - depth=test_case.depth, - ) - - assert node_ids(result.nodes) == test_case.expected_node_ids - assert edge_ids(result.edges) == test_case.expected_edge_ids - - -@pytest.mark.parametrize( - "test_case", - [ - LineageSelectorDepthErrorTestCase( - description="rejects tag selector with depth", - select=("tag:marts+",), - expected_error_fragment=( - "--depth requires name, source, seed, or path-between selectors" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_unclear_selector_anchor_with_depth_when_selecting_then_raises_user_error( - test_case: LineageSelectorDepthErrorTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - - with pytest.raises(CliUserError) as error: - select_selector_lineage( - graph=graph, - select=test_case.select, - exclude=(), - depth=1, - ) - - assert test_case.expected_error_fragment in str(error.value) - - -@pytest.mark.parametrize( - "test_case", - [ - LineagePathSelectorTestCase( - description="keeps only nodes on paths between endpoints", - select=("raw_orders~daily_rollup",), - expected_node_ids=( - "model:daily_rollup", - "model:fact_orders", - "model:stg_orders", - "source:raw_orders", - ), - ), - LineagePathSelectorTestCase( - description="adds upstreams of the path start", - select=("+stg_orders~daily_rollup",), - expected_node_ids=( - "model:daily_rollup", - "model:fact_orders", - "model:stg_orders", - "source:raw_orders", - ), - ), - LineagePathSelectorTestCase( - description="adds downstreams of the path end", - select=("raw_orders~stg_orders+",), - expected_node_ids=( - "model:daily_rollup", - "model:fact_orders", - "model:stg_orders", - "source:raw_orders", - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_path_selector_when_selecting_lineage_then_returns_path_nodes( - test_case: LineagePathSelectorTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - - result: LineageGraph = select_selector_lineage( - graph=graph, - select=test_case.select, - exclude=(), - depth=None, - ) - - assert node_ids(result.nodes) == test_case.expected_node_ids - - -@pytest.mark.parametrize( - "test_case", - [ - LineagePathSelectorErrorTestCase( - description="rejects an end that is not downstream of the start", - select=("daily_rollup~raw_orders",), - expected_error_fragment=( - "'source:raw_orders' is not downstream of 'model:daily_rollup'" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_unreachable_path_selector_when_selecting_lineage_then_raises_user_error( - test_case: LineagePathSelectorErrorTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - - with pytest.raises(PlannerInputError) as error: - select_selector_lineage( - graph=graph, - select=test_case.select, - exclude=(), - depth=None, - ) - - assert test_case.expected_error_fragment in str(error.value) - - -@pytest.mark.parametrize( - "test_case", - [ - NormalizeLineageTargetTestCase( - description="bare model is unchanged", - target="stg_orders", - expected_target="stg_orders", - ), - NormalizeLineageTargetTestCase( - description="model prefix is stripped", - target="model:stg_orders", - expected_target="stg_orders", - ), - NormalizeLineageTargetTestCase( - description="model prefix on a column keeps the column", - target="model:stg_orders.amount", - expected_target="stg_orders.amount", - ), - NormalizeLineageTargetTestCase( - description="column prefix is stripped", - target="column:stg_orders.amount", - expected_target="stg_orders.amount", - ), - NormalizeLineageTargetTestCase( - description="source prefix is stripped", - target="source:raw_orders", - expected_target="raw_orders", - ), - ], - ids=lambda case: case.description, -) -def test_given_target_when_normalizing_then_strips_matching_kind_prefix( - test_case: NormalizeLineageTargetTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - - assert ( - normalize_lineage_target(graph=graph, target=test_case.target) == test_case.expected_target - ) - - -@pytest.mark.parametrize( - "test_case", - [ - NormalizeLineageTargetErrorTestCase( - description="prefix naming another kind is unknown", - target="source:stg_orders", - expected_code="C305", - ), - NormalizeLineageTargetErrorTestCase( - description="column prefix without a column is unknown", - target="column:stg_orders", - expected_code="C305", - ), - NormalizeLineageTargetErrorTestCase( - description="column prefix on a source column is unknown", - target="column:raw_orders.id", - expected_code="C305", - ), - NormalizeLineageTargetErrorTestCase( - description="prefixed missing model is unknown", - target="model:missing", - expected_code="C305", - ), - ], - ids=lambda case: case.description, -) -def test_given_mismatched_kind_prefix_when_normalizing_then_raises_unknown_target( - test_case: NormalizeLineageTargetErrorTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - - with pytest.raises(CliUserError) as raised: - _ = normalize_lineage_target(graph=graph, target=test_case.target) - - assert raised.value.code == test_case.expected_code - - -@pytest.mark.parametrize( - "test_case", - ( - SharedSelectorParityTestCase( - description="models-rooted path selects one folder", - select=("path:models/staging",), - exclude=(), - expected_node_ids=("model:stg_orders",), - ), - SharedSelectorParityTestCase( - description="models root selects every model", - select=("path:models",), - exclude=(), - expected_node_ids=("model:daily_rollup", "model:fact_orders", "model:stg_orders"), - ), - SharedSelectorParityTestCase( - description="bare models-rooted path is a path selector", - select=("models/marts",), - exclude=(), - expected_node_ids=("model:daily_rollup", "model:fact_orders"), - ), - SharedSelectorParityTestCase( - description="path selector expands upstream", - select=("+path:models/staging",), - exclude=(), - expected_node_ids=("model:stg_orders", "source:raw_orders"), - ), - SharedSelectorParityTestCase( - description="name glob matches every resource kind", - select=("*_orders",), - exclude=(), - expected_node_ids=("model:fact_orders", "model:stg_orders", "source:raw_orders"), - ), - SharedSelectorParityTestCase( - description="typed glob matches only that kind", - select=("source:raw_*",), - exclude=(), - expected_node_ids=("source:raw_orders",), - ), - SharedSelectorParityTestCase( - description="seed selector expands downstream", - select=("seed:waffle_types+",), - exclude=(), - expected_node_ids=("model:daily_rollup", "model:fact_orders", "seed:waffle_types"), - ), - SharedSelectorParityTestCase( - description="exclude removes a resolved upstream", - select=("+fact_orders",), - exclude=("source:raw_orders",), - expected_node_ids=("model:fact_orders", "model:stg_orders", "seed:waffle_types"), - ), - SharedSelectorParityTestCase( - description="comma intersects tag and path", - select=("tag:marts,path:models/marts",), - exclude=(), - expected_node_ids=("model:daily_rollup", "model:fact_orders"), - ), - SharedSelectorParityTestCase( - description="space-separated selectors are unioned", - select=("stg_orders daily_rollup",), - exclude=(), - expected_node_ids=("model:daily_rollup", "model:stg_orders"), - ), - SharedSelectorParityTestCase( - description="path-between keeps nodes on the path", - select=("raw_orders~fact_orders",), - exclude=(), - expected_node_ids=("model:fact_orders", "model:stg_orders", "source:raw_orders"), - ), - ), - ids=lambda case: case.description, -) -def test_given_selector_when_selecting_lineage_then_matches_shared_project_selection( - test_case: SharedSelectorParityTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - - lineage: LineageGraph = select_selector_lineage( - graph=graph, - select=test_case.select, - exclude=test_case.exclude, - depth=None, - ) - shared: frozenset[CompiledObjectKey] = resolve_project_selectors( - select=test_case.select, - exclude=test_case.exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, - ) - - assert node_ids(lineage.nodes) == test_case.expected_node_ids - assert tuple(sorted(f"{key.resource_type}:{key.name}" for key in shared)) == ( - test_case.expected_node_ids - ) - - -@pytest.mark.parametrize( - "test_case", - ( - SharedSelectorErrorParityTestCase( - description="path without models root is rejected", - select=("path:staging",), - expected_code="S012", - ), - SharedSelectorErrorParityTestCase( - description="unknown models-rooted path is rejected", - select=("path:models/missing",), - expected_code="S009", - ), - SharedSelectorErrorParityTestCase( - description="unknown tag is rejected", - select=("tag:missing",), - expected_code="S008", - ), - SharedSelectorErrorParityTestCase( - description="unknown name is rejected", - select=("missing_model",), - expected_code="S007", - ), - SharedSelectorErrorParityTestCase( - description="unsupported selector kind is rejected", - select=("model:fact_orders",), - expected_code="S005", - ), - SharedSelectorErrorParityTestCase( - description="inner plus marker is rejected", - select=("stg+orders",), - expected_code="S002", - ), - ), - ids=lambda case: case.description, -) -def test_given_invalid_selector_when_selecting_lineage_then_raises_shared_selector_error( - test_case: SharedSelectorErrorParityTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - - with pytest.raises(PlannerInputError) as lineage_error: - _ = select_selector_lineage(graph=graph, select=test_case.select, exclude=(), depth=None) - with pytest.raises(PlannerInputError) as shared_error: - _ = resolve_project_selectors( - select=test_case.select, - exclude=(), - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, - ) - - assert lineage_error.value.code == test_case.expected_code - assert shared_error.value.code == test_case.expected_code - assert str(lineage_error.value) == str(shared_error.value) - - -@pytest.mark.parametrize( - "test_case", - [ - SharedSelectorDepthTestCase( - description="glob anchors every match when trimming to depth", - select=("*_orders+",), - direction=None, - expected_node_ids=( - "model:daily_rollup", - "model:fact_orders", - "model:stg_orders", - "source:raw_orders", - ), - ), - SharedSelectorDepthTestCase( - description="models-rooted path expands by direction", - select=("path:models/staging",), - direction="downstream", - expected_node_ids=("model:fact_orders", "model:stg_orders"), - ), - ], - ids=lambda case: case.description, -) -def test_given_shared_selector_with_depth_when_selecting_lineage_then_trims_from_matches( - test_case: SharedSelectorDepthTestCase, -) -> None: - graph: ProjectGraph = build_lineage_test_graph() - - result: LineageGraph = select_selector_lineage( - graph=graph, - select=test_case.select, - exclude=(), - depth=1, - direction=test_case.direction, - ) - - assert node_ids(result.nodes) == test_case.expected_node_ids diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py deleted file mode 100644 index 05bdafb38c..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Refactor command output tests.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py deleted file mode 100644 index 05a0405892..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Test case types for refactor output tests.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.compiler.refactoring.types import RefactorOperation, RefactorStatus - - -@dataclass(frozen=True) -class RefactorStatusLineTestCase: - description: str - status: RefactorStatus - changed_files: int - expected_line: str - - -@dataclass(frozen=True) -class RefactorTargetTestCase: - description: str - command: str - target: str - cascade: bool - expected_operation: RefactorOperation - expected_model_name: str - expected_column_name: str | None - - -@dataclass(frozen=True) -class RefactorTargetErrorTestCase: - description: str - command: str - target: str - cascade: bool - expected_message: str - expected_help: str | None - - -@dataclass(frozen=True) -class LayerMoveSuggestionTestCase: - description: str - old_path: str - new_name: str - codes: tuple[str, ...] - expected_command: str | None - expected_folder: str | None - - -@dataclass(frozen=True) -class CommandLineQuotingTestCase: - description: str - os_name: str - project_dir: str - expected_command: str - - -@dataclass(frozen=True) -class LayerFolderMappingTestCase: - description: str - expected_layers: frozenset[str] - - -@dataclass(frozen=True) -class ImportBoundaryTestCase: - description: str - imported_name: str - expected_modules: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/helpers.py deleted file mode 100644 index 8b20e1afbe..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/helpers.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Refactor output test builders.""" - -from __future__ import annotations - -import ast -from itertools import compress -from pathlib import Path, PurePosixPath - -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity -from sqlbuild.compiler.refactoring.models import FileChange, RefactorPlan, RefactorRequest -from sqlbuild.compiler.refactoring.types import RefactorOperation - - -def build_move_plan(*, changed_files: int) -> RefactorPlan: - """Return a move plan that changes the given number of files.""" - - return RefactorPlan( - request=RefactorRequest( - operation=RefactorOperation.MOVE_MODEL, - model_name="orders", - new_name="", - destination="models/marts/orders.sql", - ), - changes=tuple( - FileChange( - path=f"models/orders_{index}.sql", original_path=f"models/orders_{index}.sql" - ) - for index in range(changed_files) - ), - ) - - -def build_rename_plan(*, old_path: str, new_name: str) -> RefactorPlan: - """Return a model rename plan whose file keeps its folder.""" - - return RefactorPlan( - request=RefactorRequest( - operation=RefactorOperation.RENAME_MODEL, - model_name=PurePosixPath(old_path).stem, - new_name=new_name, - ), - changes=( - FileChange( - path=PurePosixPath(old_path).with_name(f"{new_name}.sql").as_posix(), - original_path=old_path, - ), - ), - ) - - -def rule_diagnostics(*, codes: tuple[str, ...], path: str) -> tuple[CompilerDiagnostic, ...]: - """Return one compile error per rule code, reported on `path`.""" - - return tuple( - CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=code, - message=f"{code} failed", - path=Path(path), - ) - for code in codes - ) - - -def modules_importing(*, package_dir: Path, name: str) -> list[str]: - """Return the sorted file names of the package's modules that import `name`.""" - - paths: list[Path] = sorted(package_dir.glob("*.py")) - flags: list[bool] = [_imports(path=path, name=name) for path in paths] - return [path.name for path in compress(paths, flags)] - - -def _imports(*, path: Path, name: str) -> bool: - tree: ast.Module = ast.parse(path.read_text(encoding="utf-8")) - imported: set[tuple[type[ast.AST], object]] = { - (type(node), getattr(node, "name", None)) for node in ast.walk(tree) - } - return (ast.alias, name) in imported diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_facts_boundary.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_facts_boundary.py deleted file mode 100644 index b5ee4a8b80..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_facts_boundary.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Only the facts module of `sqb rename` and `sqb mv` reads `CompiledProject`.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -import sqlbuild.cli.commands._helpers.refactor as refactor_package -from tests.unit.src.sqlbuild.cli.commands._helpers.refactor._test_types import ( - ImportBoundaryTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.refactor.helpers import modules_importing - - -@pytest.mark.parametrize( - "test_case", - [ - ImportBoundaryTestCase( - description="the native command core will return these facts instead", - imported_name="CompiledProjectView", - expected_modules=("facts.py",), - ), - ], - ids=lambda case: case.description, -) -def test_given_refactor_shell_when_reading_imports_then_only_facts_module_reads_compiled_project( - test_case: ImportBoundaryTestCase, -) -> None: - package_dir: Path = Path(refactor_package.__file__).parent - - importing: list[str] = modules_importing(package_dir=package_dir, name=test_case.imported_name) - - assert tuple(importing) == test_case.expected_modules - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_layer_move.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_layer_move.py deleted file mode 100644 index be5696ecae..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_layer_move.py +++ /dev/null @@ -1,195 +0,0 @@ -"""The one-step `sqb mv` suggested when a model rename fails only on the layer-folder rules.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.cli.commands._helpers.refactor.layer_move import layer_move_suggestion -from sqlbuild.cli.commands.constants import CONTIGUOUS_LAYER_FOLDERS, LAYER_FOLDERS -from sqlbuild.cli.commands.models import LayerMoveSuggestion -from sqlbuild.compiler.refactoring.models import RefactorPlan -from tests.unit.src.sqlbuild.cli.commands._helpers.refactor._test_types import ( - CommandLineQuotingTestCase, - LayerFolderMappingTestCase, - LayerMoveSuggestionTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.refactor.helpers import ( - build_rename_plan, - rule_diagnostics, -) - -_LAYER_CODES: tuple[str, str] = ("SQBRPROJECT102", "SQBRPROJECT105") - - -@pytest.mark.parametrize( - "test_case", - [ - LayerMoveSuggestionTestCase( - description="int_clean to int_enriched keeps domain and sub-folders", - old_path="models/orders/intermediate/clean/web/orders__int_clean__lines__web.sql", - new_name="orders__int_enriched__lines__web", - codes=_LAYER_CODES, - expected_command=( - "sqb mv orders__int_clean__lines__web " - "models/orders/intermediate/enriched/web/orders__int_enriched__lines__web.sql" - ), - expected_folder="models/orders/intermediate/enriched", - ), - LayerMoveSuggestionTestCase( - description="staging to a mart view replaces the one-segment layer folder", - old_path="models/customers/staging/customers__stg__accounts.sql", - new_name="customers__mart_v__accounts", - codes=("SQBRPROJECT105",), - expected_command=( - "sqb mv customers__stg__accounts models/customers/mart/customers__mart_v__accounts.sql" - ), - expected_folder="models/customers/mart", - ), - LayerMoveSuggestionTestCase( - description="a bare intermediate folder becomes the int_clean folder", - old_path="models/orders/intermediate/orders__int_v__totals.sql", - new_name="orders__int_clean__totals", - codes=("SQBRPROJECT102",), - expected_command=( - "sqb mv orders__int_v__totals " - "models/orders/intermediate/clean/orders__int_clean__totals.sql" - ), - expected_folder="models/orders/intermediate/clean", - ), - LayerMoveSuggestionTestCase( - description="no layer folder in the current path uses a placeholder", - old_path="models/orders/web/order_lines.sql", - new_name="orders__stg__order_lines", - codes=_LAYER_CODES, - expected_command="sqb mv order_lines 'models//orders__stg__order_lines.sql'", - expected_folder=None, - ), - LayerMoveSuggestionTestCase( - description="only the current layer's own folder is replaced", - old_path="models/mart/orders/staging/orders__stg__order_lines_x.sql", - new_name="orders__int_clean__order_lines_x", - codes=("SQBRPROJECT102",), - expected_command=( - "sqb mv orders__stg__order_lines_x " - "models/mart/orders/intermediate/clean/orders__int_clean__order_lines_x.sql" - ), - expected_folder="models/mart/orders/intermediate/clean", - ), - LayerMoveSuggestionTestCase( - description="a path without the current layer's folder uses a placeholder", - old_path="models/orders/mart/web/orders__stg__order_lines.sql", - new_name="orders__int_clean__order_lines", - codes=_LAYER_CODES, - expected_command=( - "sqb mv orders__stg__order_lines 'models//orders__int_clean__order_lines.sql'" - ), - expected_folder=None, - ), - LayerMoveSuggestionTestCase( - description="a layer without a folder mapping uses a placeholder", - old_path="models/orders/intermediate/clean/orders__int_clean__totals.sql", - new_name="orders__int_archive__totals", - codes=_LAYER_CODES, - expected_command=( - "sqb mv orders__int_clean__totals 'models//orders__int_archive__totals.sql'" - ), - expected_folder=None, - ), - LayerMoveSuggestionTestCase( - description="another rule failing as well gets no suggestion", - old_path="models/orders/intermediate/clean/orders__int_clean__totals.sql", - new_name="orders__int_enriched__totals", - codes=("SQBRPROJECT105", "SQBRMODEL103"), - expected_command=None, - expected_folder=None, - ), - ], - ids=lambda case: case.description, -) -def test_given_rename_compile_errors_when_suggesting_layer_move_then_one_step_mv_is_exact( - test_case: LayerMoveSuggestionTestCase, -) -> None: - plan: RefactorPlan = build_rename_plan(old_path=test_case.old_path, new_name=test_case.new_name) - - suggestion: LayerMoveSuggestion | None = layer_move_suggestion( - plan=plan, - diagnostics=rule_diagnostics(codes=test_case.codes, path=plan.changes[0].path), - project_dir=None, - ) - - assert ( - getattr(suggestion, "command", None), - getattr(suggestion, "folder", None), - ) == (test_case.expected_command, test_case.expected_folder) - - -@pytest.mark.parametrize( - "test_case", - [ - CommandLineQuotingTestCase( - description="posix shells get POSIX quoting", - os_name="posix", - project_dir="/srv/order projects/orders", - expected_command=( - "sqb mv orders__int_clean__lines__web " - "models/orders/intermediate/enriched/web/orders__int_enriched__lines__web.sql " - "--project-dir '/srv/order projects/orders'" - ), - ), - CommandLineQuotingTestCase( - description="windows gets cmd.exe quoting", - os_name="nt", - project_dir="C:\\order projects\\orders", - expected_command=( - "sqb mv orders__int_clean__lines__web " - "models/orders/intermediate/enriched/web/orders__int_enriched__lines__web.sql " - '--project-dir "C:\\order projects\\orders"' - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_platform_when_suggesting_layer_move_then_command_is_quoted_for_its_shell( - test_case: CommandLineQuotingTestCase, -) -> None: - plan: RefactorPlan = build_rename_plan( - old_path="models/orders/intermediate/clean/web/orders__int_clean__lines__web.sql", - new_name="orders__int_enriched__lines__web", - ) - - suggestion: LayerMoveSuggestion | None = layer_move_suggestion( - plan=plan, - diagnostics=rule_diagnostics(codes=_LAYER_CODES, path=plan.changes[0].path), - project_dir=Path(test_case.project_dir), - os_name=test_case.os_name, - ) - - assert getattr(suggestion, "command", None) == test_case.expected_command - - -@pytest.mark.parametrize( - "test_case", - [ - LayerFolderMappingTestCase( - description="every parse_name layer, mirroring model_layers.rs and domain_layout.rs", - expected_layers=frozenset( - {"stg", "stg_v", "int_clean", "int_v", "int_enriched", "mart", "mart_v"} - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_rule_grammar_layers_when_reading_layer_folders_then_each_layer_has_both_folders( - test_case: LayerFolderMappingTestCase, -) -> None: - """LAYER_FOLDERS mirrors model_layers.rs (SQBRPROJECT105); the other, domain_layout.rs (102).""" - assert (frozenset(LAYER_FOLDERS), frozenset(CONTIGUOUS_LAYER_FOLDERS)) == ( - test_case.expected_layers, - test_case.expected_layers, - ) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_output.py deleted file mode 100644 index d868f9b4d4..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_output.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Terminal status line of `sqb rename` and `sqb mv` text output.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.refactor.output import render_refactor_text -from sqlbuild.compiler.refactoring.types import RefactorStatus -from tests.unit.src.sqlbuild.cli.commands._helpers.refactor._test_types import ( - RefactorStatusLineTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.refactor.helpers import build_move_plan - - -@pytest.mark.parametrize( - "test_case", - [ - RefactorStatusLineTestCase( - description="applied one file is singular", - status=RefactorStatus.APPLIED, - changed_files=1, - expected_line="Compiled: ok, 1 file changed", - ), - RefactorStatusLineTestCase( - description="applied several files is plural", - status=RefactorStatus.APPLIED, - changed_files=3, - expected_line="Compiled: ok, 3 files changed", - ), - RefactorStatusLineTestCase( - description="dry run one file is singular", - status=RefactorStatus.DRY_RUN, - changed_files=1, - expected_line="Dry run: compiles, 1 file would change; nothing written", - ), - RefactorStatusLineTestCase( - description="dry run several files is plural", - status=RefactorStatus.DRY_RUN, - changed_files=2, - expected_line="Dry run: compiles, 2 files would change; nothing written", - ), - ], - ids=lambda case: case.description, -) -def test_given_changed_file_count_when_rendering_refactor_text_then_status_line_agrees_in_number( - test_case: RefactorStatusLineTestCase, -) -> None: - output: str = render_refactor_text( - plan=build_move_plan(changed_files=test_case.changed_files), - status=test_case.status, - diagnostics=(), - use_color=False, - ) - - assert output.splitlines()[-1] == test_case.expected_line - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_target.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_target.py deleted file mode 100644 index d1be6aef66..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_target.py +++ /dev/null @@ -1,232 +0,0 @@ -"""Reading `sqb rename` and `sqb mv` targets, with and without kind prefixes.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.refactor.target import refactor_request_for_target -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import RefactorCommandRequest -from sqlbuild.cli.commands.types import CliCommand -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.refactoring.models import RefactorRequest -from sqlbuild.compiler.refactoring.types import RefactorOperation -from tests.unit.src.sqlbuild.cli.commands._helpers.refactor._test_types import ( - RefactorTargetErrorTestCase, - RefactorTargetTestCase, -) - -_ALL_KEYS: dict[str, CompiledObjectKey] = { - "stg_orders": CompiledObjectKey(CompiledResourceType.MODEL, "stg_orders"), - "raw_orders": CompiledObjectKey(CompiledResourceType.SOURCE, "raw_orders"), - "order_types": CompiledObjectKey(CompiledResourceType.SEED, "order_types"), - "is_completed": CompiledObjectKey(CompiledResourceType.UDF, "is_completed"), -} -_RENAME_HELP: str = ( - "write the target as or .; the model: and column: prefixes are optional" -) - - -@pytest.mark.parametrize( - "test_case", - [ - RefactorTargetTestCase( - description="bare model renames the model", - command=CliCommand.RENAME, - target="stg_orders", - cascade=False, - expected_operation=RefactorOperation.RENAME_MODEL, - expected_model_name="stg_orders", - expected_column_name=None, - ), - RefactorTargetTestCase( - description="bare model column renames the column", - command=CliCommand.RENAME, - target="stg_orders.amount", - cascade=True, - expected_operation=RefactorOperation.RENAME_COLUMN, - expected_model_name="stg_orders", - expected_column_name="amount", - ), - RefactorTargetTestCase( - description="model prefix renames the model", - command=CliCommand.RENAME, - target="model:stg_orders", - cascade=False, - expected_operation=RefactorOperation.RENAME_MODEL, - expected_model_name="stg_orders", - expected_column_name=None, - ), - RefactorTargetTestCase( - description="column prefix renames the column", - command=CliCommand.RENAME, - target="column:stg_orders.amount", - cascade=False, - expected_operation=RefactorOperation.RENAME_COLUMN, - expected_model_name="stg_orders", - expected_column_name="amount", - ), - RefactorTargetTestCase( - description="bare model moves the model", - command=CliCommand.MV, - target="stg_orders", - cascade=False, - expected_operation=RefactorOperation.MOVE_MODEL, - expected_model_name="stg_orders", - expected_column_name=None, - ), - RefactorTargetTestCase( - description="model prefix moves the model", - command=CliCommand.MV, - target="model:stg_orders", - cascade=False, - expected_operation=RefactorOperation.MOVE_MODEL, - expected_model_name="stg_orders", - expected_column_name=None, - ), - RefactorTargetTestCase( - description="unknown bare name is left for the planner to report", - command=CliCommand.RENAME, - target="missing_model", - cascade=False, - expected_operation=RefactorOperation.RENAME_MODEL, - expected_model_name="missing_model", - expected_column_name=None, - ), - ], - ids=lambda case: case.description, -) -def test_given_target_when_reading_then_returns_matching_request( - test_case: RefactorTargetTestCase, -) -> None: - result: RefactorRequest = refactor_request_for_target( - request=RefactorCommandRequest( - command=test_case.command, - target=test_case.target, - new_name="renamed", - destination="models/marts/", - cascade=test_case.cascade, - ), - all_keys=_ALL_KEYS, - ) - - assert result.operation == test_case.expected_operation - assert result.model_name == test_case.expected_model_name - assert result.column_name == test_case.expected_column_name - - -@pytest.mark.parametrize( - "test_case", - [ - RefactorTargetErrorTestCase( - description="bare source is refused and named", - command=CliCommand.RENAME, - target="raw_orders", - cascade=False, - expected_message=( - "cannot rename 'raw_orders': raw_orders is a source; " - "sqb rename renames models and model columns" - ), - expected_help=_RENAME_HELP, - ), - RefactorTargetErrorTestCase( - description="source column is refused and named", - command=CliCommand.RENAME, - target="raw_orders.id", - cascade=False, - expected_message=( - "cannot rename 'raw_orders.id': raw_orders is a source; " - "sqb rename renames models and model columns" - ), - expected_help=_RENAME_HELP, - ), - RefactorTargetErrorTestCase( - description="seed is refused even with its own prefix", - command=CliCommand.RENAME, - target="seed:order_types", - cascade=False, - expected_message=( - "cannot rename 'seed:order_types': order_types is a seed; " - "sqb rename renames models and model columns" - ), - expected_help=_RENAME_HELP, - ), - RefactorTargetErrorTestCase( - description="function cannot be moved", - command=CliCommand.MV, - target="is_completed", - cascade=False, - expected_message=( - "cannot move 'is_completed': is_completed is a udf; sqb mv moves models" - ), - expected_help="write the target as ; the model: prefix is optional", - ), - RefactorTargetErrorTestCase( - description="prefix that contradicts the resource names what it is", - command=CliCommand.RENAME, - target="source:stg_orders", - cascade=False, - expected_message="cannot rename 'source:stg_orders': stg_orders is a model, not a source", - expected_help=_RENAME_HELP, - ), - RefactorTargetErrorTestCase( - description="column prefix without a column is refused", - command=CliCommand.RENAME, - target="column:stg_orders", - cascade=False, - expected_message=( - "cannot rename 'column:stg_orders': column: needs ., " - "and stg_orders names a model" - ), - expected_help=_RENAME_HELP, - ), - RefactorTargetErrorTestCase( - description="empty column name is refused", - command=CliCommand.RENAME, - target="stg_orders.", - cascade=False, - expected_message="cannot rename 'stg_orders.'", - expected_help=_RENAME_HELP, - ), - RefactorTargetErrorTestCase( - description="moving a column points at sqb rename", - command=CliCommand.MV, - target="stg_orders.amount", - cascade=False, - expected_message="sqb mv moves models; 'stg_orders.amount' names a column", - expected_help="rename a column with sqb rename stg_orders.amount ", - ), - RefactorTargetErrorTestCase( - description="cascade on a bare model is refused", - command=CliCommand.RENAME, - target="stg_orders", - cascade=True, - expected_message="--cascade applies to column renames only", - expected_help=None, - ), - ], - ids=lambda case: case.description, -) -def test_given_unsupported_target_when_reading_then_raises_c954( - test_case: RefactorTargetErrorTestCase, -) -> None: - with pytest.raises(CliUserError) as raised: - _ = refactor_request_for_target( - request=RefactorCommandRequest( - command=test_case.command, - target=test_case.target, - new_name="renamed", - destination="models/marts/", - cascade=test_case.cascade, - ), - all_keys=_ALL_KEYS, - ) - - assert raised.value.code == "C954" - assert raised.value.message == test_case.expected_message - assert raised.value.help == test_case.expected_help - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py index 8ff4420324..6b16cb4a26 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py @@ -4,16 +4,16 @@ import pytest -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.cli.commands._helpers.runtime import connection as connection_core -from sqlbuild.cli.commands._helpers.runtime.connection import ( +from sqlbuild._native import ( resolve_connection_config, resolve_project_connection_config, resolve_target_connection_config, ) +from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.integrations.dbt.main.profile import profile_connection from sqlbuild.integrations.dbt.models import NormalizedDbtProfileConnection from sqlbuild.spec.contracts.models import ( LocalConfig, @@ -274,7 +274,7 @@ def resolve_raw_dbt_profile_connection(**kwargs: object) -> NormalizedDbtProfile ) monkeypatch.setattr( - connection_core, + profile_connection, "resolve_raw_dbt_profile_connection", resolve_raw_dbt_profile_connection, ) @@ -333,7 +333,7 @@ def resolve_raw_dbt_profile_connection(**kwargs: object) -> NormalizedDbtProfile ) monkeypatch.setattr( - connection_core, + profile_connection, "resolve_raw_dbt_profile_connection", resolve_raw_dbt_profile_connection, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index 273490db96..efe524a245 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -6,10 +6,9 @@ import pytest +from sqlbuild._native import describe_connection_warehouse, resolve_project_connection_config from sqlbuild.cli.commands._helpers.entry.parsing import parse_cli_invocation -from sqlbuild.cli.commands._helpers.runtime.connection import resolve_project_connection_config from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope -from sqlbuild.cli.commands._helpers.runtime.warehouses import describe_connection_warehouse from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CliCommand diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_runner.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_runner.py index 9cc4fc7c37..68c3a5ab28 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_runner.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_runner.py @@ -6,9 +6,9 @@ import pytest +from sqlbuild._native import NativeProgressProjector from sqlbuild.cli.commands._helpers.scenario_execution import runner from sqlbuild.cli.commands.models import ScenarioTestCommandRequest -from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope from tests.unit.src.sqlbuild.cli.commands._helpers.scenario_execution._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py index 68bff72036..7f8d8e2281 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py @@ -65,3 +65,9 @@ def bare_target_scope_lookup() -> ScopeLookup: ), ) return build_scope_lookup(index=replace(index, resources=(*index.resources, *extra))) + + +def bare_target_scope_index(**_options: object) -> ScopeIndex: + """Load the bare-target fixture index as `sqb scope` loads an offline index.""" + + return bare_target_scope_lookup().index diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_output.py index 414b4dfa5b..ef798251b1 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_output.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild.cli.commands._helpers.scope.command import run_scope_command -from sqlbuild.cli.commands._helpers.scope.output import render_scope_result +from sqlbuild._native import render_scope_result, run_scope_command from sqlbuild.cli.commands.models import ScopeCommandRequest from tests.unit.src.sqlbuild.cli.commands._helpers.scope._test_types import ( ScopeColourCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py index adf0b3cc22..769ad2aac9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py @@ -2,15 +2,18 @@ from __future__ import annotations +from io import StringIO + import pytest -from sqlbuild.cli.commands._helpers.scope.target import qualify_scope_target +from sqlbuild._native import run_scope_command from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.cli.commands.models import ScopeCommandRequest from tests.unit.src.sqlbuild.cli.commands._helpers.scope._test_types import ( ScopeTargetCase, ScopeTargetErrorCase, ) -from tests.unit.src.sqlbuild.cli.commands._helpers.scope.helpers import bare_target_scope_lookup +from tests.unit.src.sqlbuild.cli.commands._helpers.scope.helpers import bare_target_scope_index @pytest.mark.parametrize( @@ -23,22 +26,21 @@ "bare function", "udf__is_completed_order", "function:udf__is_completed_order" ), ScopeTargetCase("prefixed model unchanged", "model:orders", "model:orders"), - ScopeTargetCase("prefixed enum unchanged", "enum:order_status", "enum:order_status"), - ScopeTargetCase( - "resource path unchanged", "models/staging/orders.sql", "models/staging/orders.sql" - ), - ScopeTargetCase("prospective scope has no target", None, None), ), ids=lambda case: case.description, ) def test_given_scope_target_when_qualifying_then_bare_resources_gain_their_kind( test_case: ScopeTargetCase, ) -> None: - assert ( - qualify_scope_target(lookup=bare_target_scope_lookup(), target=test_case.target) - == test_case.expected_target + stream: StringIO = StringIO() + _ = run_scope_command( + request=ScopeCommandRequest(target=test_case.target), + load_scope_index=bare_target_scope_index, + output_stream=stream, ) + assert f" Target: {test_case.expected_target}\n" in stream.getvalue() + @pytest.mark.parametrize( "test_case", @@ -72,7 +74,11 @@ def test_given_ambiguous_or_prefix_only_bare_name_when_qualifying_then_raises_cl test_case: ScopeTargetErrorCase, ) -> None: with pytest.raises(CliUserError) as error: - qualify_scope_target(lookup=bare_target_scope_lookup(), target=test_case.target) + _ = run_scope_command( + request=ScopeCommandRequest(target=test_case.target), + load_scope_index=bare_target_scope_index, + output_stream=StringIO(), + ) assert error.value.code == "C960" assert error.value.message == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py index 3b8a856999..2005914a88 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py @@ -223,7 +223,7 @@ def run_sqlbuild_debug( debug_dbt_from_project, ) monkeypatch.setattr( - "sqlbuild.cli.commands.main.dbt._dbt_debug.run_sqlbuild_debug", run_sqlbuild_debug + "sqlbuild.cli.commands.main.dbt._dbt_debug.run_debug_command", run_sqlbuild_debug ) def ensure_existing_sqlbuild_project( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index edbb12303b..34d5834104 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, Protocol, cast +from sqlbuild._native import NativeProgressProjector from sqlbuild.cli.entry.models import ( CliEntrypointHandlers, ) @@ -88,3 +89,16 @@ def build_handlers(**overrides: Any) -> CliEntrypointHandlers: run_rules=overrides.get("run_rules", noop_handler), run_scope=overrides.get("run_scope", noop_handler), ) + + +class CloseFailingProjector: + """Delegate to a native progress projector whose close always fails.""" + + def __init__(self, **options: Any) -> None: + self._projector: NativeProgressProjector = NativeProgressProjector(**options) + + def __getattr__(self, name: str) -> object: + return getattr(self._projector, name) + + def close(self) -> None: + raise OSError("controlled projector close failure") diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py index b50d53b007..0d920969dc 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py @@ -10,19 +10,24 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild.cli.commands._helpers.entry.observability import cli_observability_scope -from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace -from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV -from sqlbuild.cli.progress.classes.native_progress_projector import ( +from sqlbuild._native import ( NativeProgressProjector, current_native_progress_projector, ) +from sqlbuild.cli.commands._helpers.entry.observability import cli_observability_scope +from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace +from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV from sqlbuild.observability import OperationLifecycle from tests.unit.src.sqlbuild.cli.commands.main.entry._test_types import ( MachineProgressRoutingCase, NoExporterFastPathTestCase, ObservabilityCleanupCase, ) +from tests.unit.src.sqlbuild.cli.commands.main.entry.helpers import CloseFailingProjector + +_PROJECTOR_FACTORY: str = ( + "sqlbuild.cli.commands._helpers.entry.observability.NativeProgressProjector" +) @pytest.mark.parametrize( @@ -44,11 +49,7 @@ def test_given_projector_close_failure_when_command_fails_then_original_error_an outer_token: Token[NativeProgressProjector | None] = outer.install() with ( - patch.object( - NativeProgressProjector, - "close", - side_effect=OSError("controlled projector close failure"), - ), + patch(_PROJECTOR_FACTORY, CloseFailingProjector), pytest.raises(ValueError, match=test_case.expected_original_error), ): with cli_observability_scope(args=args, project_dir=tmp_path): @@ -203,11 +204,7 @@ def test_given_projector_close_failure_when_scope_succeeds_then_cleanup_does_not ) -> None: args: CliNamespace = CliNamespace() - with patch.object( - NativeProgressProjector, - "close", - side_effect=OSError("controlled projector close failure"), - ): + with patch(_PROJECTOR_FACTORY, CloseFailingProjector): with cli_observability_scope(args=args, project_dir=tmp_path): result: str = "command result" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py index 9276cb2d44..c053d7e024 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py @@ -1,13 +1,6 @@ from __future__ import annotations from dataclasses import dataclass -from datetime import datetime - - -@dataclass(frozen=True) -class FreshnessClockTestCase: - description: str - expected_observed_at: datetime @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_freshness.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_freshness.py deleted file mode 100644 index aa7d441bd9..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_freshness.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime, timedelta, timezone, tzinfo - -import pytest - -from sqlbuild.cli.commands.main.execution import _freshness -from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import FreshnessClockTestCase - - -class NonUtcLocalDateTime: - non_utc_local_time: datetime = datetime( - 2026, 1, 15, 17, 30, tzinfo=timezone(timedelta(hours=5, minutes=30)) - ) - - @classmethod - def now(cls, tz: tzinfo | None = None) -> datetime: - assert tz is UTC - return datetime(2026, 1, 15, 12, 0, tzinfo=UTC) - - -@pytest.mark.parametrize( - "test_case", - ( - FreshnessClockTestCase( - description="non UTC local timezone still observes in UTC", - expected_observed_at=datetime(2026, 1, 15, 12, 0, tzinfo=UTC), - ), - ), - ids=lambda case: case.description, -) -def test_given_non_utc_local_timezone_when_observing_freshness_then_clock_returns_aware_utc( - monkeypatch: pytest.MonkeyPatch, - test_case: FreshnessClockTestCase, -) -> None: - monkeypatch.setattr(_freshness, "datetime", NonUtcLocalDateTime) - - observed_at: datetime = _freshness._source_freshness_observed_at() - - assert observed_at == test_case.expected_observed_at - assert observed_at.tzinfo is UTC - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py b/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py index 59b4f8de75..0a32625d93 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py @@ -8,9 +8,9 @@ import pytest from _pytest.capture import CaptureResult +from sqlbuild._native import write_cost_run from sqlbuild.cli.commands._helpers.cost.command import _parse_since_bound, run_cost_command from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cost.classes.run_cost_store import RunCostStore from tests.unit.src.sqlbuild.cli.commands.main.inspection.cost._test_types import ( CostCommandTestCase, CostRelativeSinceTestCase, @@ -37,11 +37,11 @@ def test_given_latest_selector_when_running_cost_then_latest_record_is_rendered( capsys: pytest.CaptureFixture[str], ) -> None: now: datetime = datetime(2026, 8, 23, tzinfo=UTC) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record(run_id="run-old", completed_at=now, usd="1"), ) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record( run_id="run-new", completed_at=now + timedelta(minutes=1), usd="2" @@ -70,7 +70,7 @@ def test_given_history_date_range_when_running_json_then_until_date_is_inclusive tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record( run_id="run-included", @@ -78,7 +78,7 @@ def test_given_history_date_range_when_running_json_then_until_date_is_inclusive usd="1", ), ) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record( run_id="run-excluded", @@ -139,11 +139,11 @@ def test_given_ambiguous_prefix_when_running_cost_then_user_error_is_raised( tmp_path: Path, ) -> None: now: datetime = datetime(2026, 8, 23, tzinfo=UTC) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record(run_id="run-123", completed_at=now, usd="1"), ) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record( run_id="run-456", completed_at=now + timedelta(seconds=1), usd="2" @@ -170,13 +170,13 @@ def test_given_relative_since_bound_when_running_history_then_old_runs_are_exclu capsys: pytest.CaptureFixture[str], ) -> None: now: datetime = datetime.now(UTC) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record( run_id="run-recent", completed_at=now - timedelta(days=6), usd="1" ), ) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record( run_id="run-old", completed_at=now - timedelta(days=8), usd="2" @@ -287,7 +287,7 @@ def test_given_json_detail_request_when_running_cost_then_stable_schema_is_emitt capsys: pytest.CaptureFixture[str], ) -> None: now: datetime = datetime(2026, 8, 23, tzinfo=UTC) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record( run_id=test_case.expected_run_ids[0], completed_at=now, usd="2" @@ -328,7 +328,7 @@ def test_given_json_detail_limit_when_running_cost_then_models_only_are_truncate capsys: pytest.CaptureFixture[str], ) -> None: now: datetime = datetime(2026, 8, 23, tzinfo=UTC) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record( run_id=test_case.expected_run_ids[0], completed_at=now, usd="2" @@ -370,7 +370,7 @@ def test_given_equal_history_metrics_when_sorting_then_completion_and_run_id_bre ("run-a", origin), ("run-new", origin + timedelta(minutes=1)), ): - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record( run_id=run_id, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py index e87261302d..6e32597495 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -1,42 +1,9 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class JanitorConfirmationInterruptTestCase: - description: str - expected_result: bool - expected_output_fragments: tuple[str, ...] - unexpected_output_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class JanitorDisabledOutputTestCase: - description: str - use_color: bool - expected_output: str | None = None - expected_prefix: str | None = None - - @dataclass(frozen=True) class JanitorPlanOutputTestCase: description: str use_color: bool expected_output_fragments: tuple[str, ...] unexpected_output_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class JanitorDirectArchiveConfirmationTestCase: - description: str - typed_response: str - expected_result: bool - expected_output_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class JanitorSettingsResolutionTestCase: - description: str - project_toml: str - cli_retention_days: int | None - expected_retention_days: int - expected_archive_retention_days: int diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 23c6540e30..c3cc3ae1e4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -1,22 +1,12 @@ -"""Tests for janitor command behavior.""" +"""Tests for the janitor plan preview.""" from __future__ import annotations -import builtins -from io import StringIO - import pytest -from sqlbuild.cli.commands._helpers.janitor_output.output import write_disabled, write_plan -from sqlbuild.cli.commands._helpers.janitor_output.outputs import confirm_janitor_plan -from sqlbuild.cli.commands.models import JanitorPlanningResult -from sqlbuild.executor.janitor.models import ( - JanitorPlan, -) +from sqlbuild._native import render_janitor_plan +from sqlbuild.executor.janitor.models import JanitorPlan from tests.unit.src.sqlbuild.cli.commands.main.janitor._test_types import ( - JanitorConfirmationInterruptTestCase, - JanitorDirectArchiveConfirmationTestCase, - JanitorDisabledOutputTestCase, JanitorPlanOutputTestCase, ) from tests.unit.src.sqlbuild.cli.commands.main.janitor.helpers import ( @@ -25,98 +15,6 @@ ) -@pytest.mark.parametrize( - "test_case", - [ - JanitorConfirmationInterruptTestCase( - description="treats keyboard interrupt at confirmation as cancellation", - expected_result=False, - expected_output_fragments=( - "Janitor will archive 1 and delete 1 objects from dev.", - "Type `archive 1 and delete 1 objects from dev` to continue: ", - ), - unexpected_output_fragments=("KeyboardInterrupt", "Traceback"), - ) - ], - ids=lambda case: case.description, -) -def test_given_janitor_confirmation_when_keyboard_interrupt_then_returns_cancelled( - test_case: JanitorConfirmationInterruptTestCase, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - plan: JanitorPlan = build_direct_archive_plan() - - def interrupting_input() -> str: - raise KeyboardInterrupt - - monkeypatch.setattr(builtins, "input", interrupting_input) - - result: bool = confirm_janitor_plan(planning_result=JanitorPlanningResult(plan=plan)) - - output: str = capsys.readouterr().out - assert result is test_case.expected_result - for fragment in test_case.expected_output_fragments: - assert fragment in output - for fragment in test_case.unexpected_output_fragments: - assert fragment not in output - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorDisabledOutputTestCase( - description="preserves disabled no-color output", - use_color=False, - expected_output="Janitor is disabled for this project.\n\n" - "Janitor is opt-in. It previews stale warehouse objects and asks before " - "archiving or deleting anything.\n\n" - "Add this block to sqlbuild_project.toml:\n\n" - "[janitor]\n" - "enabled = true\n\n" - "After enabling, run janitor again to preview cleanup:\n" - " sqb janitor\n", - ) - ], - ids=lambda case: case.description, -) -def test_given_disabled_janitor_when_writing_without_color_then_preserves_output( - test_case: JanitorDisabledOutputTestCase, -) -> None: - stream: StringIO = StringIO() - - write_disabled(stream=stream, use_color=test_case.use_color) - - assert stream.getvalue() == test_case.expected_output - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorDisabledOutputTestCase( - description="styles disabled title", - use_color=True, - expected_prefix="\033[33m\033[1mJanitor is disabled for this project.\033[0m", - ) - ], - ids=lambda case: case.description, -) -def test_given_disabled_janitor_when_writing_with_color_then_styles_title( - test_case: JanitorDisabledOutputTestCase, -) -> None: - stream: StringIO = StringIO() - - write_disabled(stream=stream, use_color=test_case.use_color) - - assert test_case.expected_prefix is not None - assert stream.getvalue().startswith(test_case.expected_prefix) - assert ( - "\033[33mJanitor is opt-in. It previews stale warehouse objects and asks before " - "archiving or deleting anything.\033[0m" - ) in stream.getvalue() - assert "\033[34msqb janitor\033[0m" in stream.getvalue() - - @pytest.mark.parametrize( "test_case", [ @@ -148,11 +46,7 @@ def test_given_janitor_plan_when_writing_without_color_then_preserves_output_fra test_case: JanitorPlanOutputTestCase, ) -> None: plan: JanitorPlan = build_janitor_plan() - stream: StringIO = StringIO() - - write_plan(plan=plan, stream=stream, use_color=test_case.use_color) - - output: str = stream.getvalue() + output: str = render_janitor_plan(plan=plan, use_color=test_case.use_color) for fragment in test_case.expected_output_fragments: assert fragment in output for fragment in test_case.unexpected_output_fragments: @@ -181,51 +75,7 @@ def test_given_janitor_plan_when_writing_with_color_then_uses_semantic_colors( test_case: JanitorPlanOutputTestCase, ) -> None: plan: JanitorPlan = build_janitor_plan() - stream: StringIO = StringIO() - - write_plan(plan=plan, stream=stream, use_color=test_case.use_color) - - output: str = stream.getvalue() - for fragment in test_case.expected_output_fragments: - assert fragment in output - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorDirectArchiveConfirmationTestCase( - description="direct archive plan requires typed archive and delete counts", - typed_response="archive 1 and delete 1 objects from dev", - expected_result=True, - expected_output_fragments=( - "Janitor will archive 1 and delete 1 objects from dev.", - "Retention: 14 days", - "Archive retention: 14 days", - "Type `archive 1 and delete 1 objects from dev` to continue: ", - ), - ), - JanitorDirectArchiveConfirmationTestCase( - description="direct archive plan requires the archive count in confirmation", - typed_response="delete 1 objects from dev", - expected_result=False, - expected_output_fragments=("Janitor will archive 1 and delete 1 objects from dev.",), - ), - ], - ids=lambda case: case.description, -) -def test_given_direct_archive_plan_when_confirming_then_requires_exact_archive_phrase( - test_case: JanitorDirectArchiveConfirmationTestCase, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(builtins, "input", lambda: test_case.typed_response) - - result: bool = confirm_janitor_plan( - planning_result=JanitorPlanningResult(plan=build_direct_archive_plan()) - ) - - output: str = capsys.readouterr().out - assert result is test_case.expected_result + output: str = render_janitor_plan(plan=plan, use_color=test_case.use_color) for fragment in test_case.expected_output_fragments: assert fragment in output @@ -256,11 +106,9 @@ def test_given_direct_archive_plan_when_confirming_then_requires_exact_archive_p def test_given_direct_archive_plan_when_writing_preview_then_lists_archive_actions( test_case: JanitorPlanOutputTestCase, ) -> None: - stream: StringIO = StringIO() - - write_plan(plan=build_direct_archive_plan(), stream=stream, use_color=test_case.use_color) - - output: str = stream.getvalue() + output: str = render_janitor_plan( + plan=build_direct_archive_plan(), use_color=test_case.use_color + ) for fragment in test_case.expected_output_fragments: assert fragment in output for fragment in test_case.unexpected_output_fragments: diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py deleted file mode 100644 index de59ca0779..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Tests for janitor settings resolution from discovered project config.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.cli.commands._helpers.janitor_runtime.invocation import ( - resolve_janitor_invocation, - resolve_janitor_settings, -) -from sqlbuild.cli.commands.models import ( - JanitorCommandRequest, - JanitorInvocation, - JanitorSettings, -) -from tests.unit.src.sqlbuild.cli.commands.main.janitor._test_types import ( - JanitorSettingsResolutionTestCase, -) - -DIRECT_PROJECT_TOML: str = ( - 'name = "orders_direct"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' -) - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorSettingsResolutionTestCase( - description="direct mode without explicit retention resolves 14", - project_toml=DIRECT_PROJECT_TOML + "\n[janitor]\nenabled = true\n", - cli_retention_days=None, - expected_retention_days=14, - expected_archive_retention_days=14, - ), - JanitorSettingsResolutionTestCase( - description="explicit retention applies to direct mode", - project_toml=DIRECT_PROJECT_TOML - + "\n[janitor]\nenabled = true\nretention_days = 9\narchive_retention_days = 2\n", - cli_retention_days=None, - expected_retention_days=9, - expected_archive_retention_days=2, - ), - ], - ids=lambda case: case.description, -) -def test_given_project_config_when_resolving_janitor_settings_then_uses_mode_specific_default( - test_case: JanitorSettingsResolutionTestCase, - tmp_path: Path, -) -> None: - (tmp_path / "sqlbuild_project.toml").write_text(test_case.project_toml, encoding="utf-8") - request: JanitorCommandRequest = JanitorCommandRequest( - project_dir=tmp_path, no_color=True, retention_days=test_case.cli_retention_days - ) - - invocation: JanitorInvocation = resolve_janitor_invocation(request=request) - settings: JanitorSettings = resolve_janitor_settings(request=request, invocation=invocation) - - assert settings.retention_days == test_case.expected_retention_days - assert settings.archive_retention_days == test_case.expected_archive_retention_days diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index 7e94354292..026803509f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -5,15 +5,11 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild.cli.commands._helpers.playground import ( - completion_output as playground_completion_output, -) -from sqlbuild.cli.commands._helpers.playground.copy import create_playground_project -from sqlbuild.cli.commands._helpers.skills.update import maintain_sqlbuild_skills +from sqlbuild._native import maintain_sqlbuild_skills, run_playground_command from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.main.workspace._playground import run_playground from sqlbuild.cli.commands.models import PlaygroundCommandRequest from sqlbuild.cli.output.models import SkillMaintenanceResult +from sqlbuild.presentation.main import supports_color as playground_completion_output from tests.unit.src.sqlbuild.cli.commands.main.playground._test_types import ( CreatePlaygroundProjectTestCase, RunPlaygroundTestCase, @@ -228,7 +224,13 @@ def test_given_missing_target_when_creating_playground_then_it_copies_clean_temp ) -> None: target_dir: Path = tmp_path / test_case.target_relative_path - create_playground_project(target_dir=target_dir, template=test_case.template) + _ = run_playground_command( + PlaygroundCommandRequest( + project_dir=tmp_path, + target_path=str(test_case.target_relative_path), + template=test_case.template, + ) + ) expected_file: Path for expected_file in test_case.expected_files: @@ -273,7 +275,11 @@ def test_given_existing_target_when_creating_playground_then_it_raises_user_erro target_dir.mkdir() with pytest.raises(CliUserError) as exc_info: - create_playground_project(target_dir=target_dir) + _ = run_playground_command( + PlaygroundCommandRequest( + project_dir=tmp_path, target_path=str(test_case.target_relative_path) + ) + ) assert test_case.expected_error_fragment in str(exc_info.value) @@ -352,7 +358,7 @@ def test_given_playground_command_when_running_then_it_prints_next_steps( tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - exit_code: int = run_playground( + exit_code: int = run_playground_command( PlaygroundCommandRequest( project_dir=tmp_path, target_path=test_case.target_path, @@ -395,7 +401,7 @@ def test_given_color_terminal_when_running_playground_command_then_it_styles_key ) -> None: monkeypatch.setattr(playground_completion_output, "supports_color", lambda: True) - exit_code: int = run_playground( + exit_code: int = run_playground_command( PlaygroundCommandRequest( project_dir=tmp_path, target_path=test_case.target_path, @@ -432,7 +438,7 @@ def test_given_wrapper_playground_when_maintaining_skills_then_nested_project_se test_case: WrapperPlaygroundSkillsTestCase, tmp_path: Path, ) -> None: - exit_code: int = run_playground( + exit_code: int = run_playground_command( PlaygroundCommandRequest( project_dir=tmp_path, target_path="orchestrator", template=test_case.template ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py index fd4320e655..f19b8b079b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py @@ -7,11 +7,11 @@ import pytest -from sqlbuild.cli.commands._helpers.runtime import adapter_context, connection -from sqlbuild.cli.commands._helpers.scope import command as scope_command -from sqlbuild.cli.commands._helpers.scope.command import run_scope_command +from sqlbuild._native import run_scope_command +from sqlbuild.cli.commands._helpers.runtime import adapter_context from sqlbuild.cli.commands.models import ScopeCommandRequest from sqlbuild.compiler.scopes.models import ScopeCompleteness, ScopeIndex +from sqlbuild.presentation.main import supports_color as color_support from tests.unit.src.sqlbuild.cli.commands.main.scope._test_types import ScopeCommandCase from tests.unit.src.sqlbuild.compiler.scopes.helpers import report_scope_lookup @@ -25,7 +25,6 @@ def test_given_offline_index_loader_when_running_then_connection_runtime_is_not_ def fail(**_kwargs: object) -> None: pytest.fail("scope must not resolve adapters or warehouse credentials") - monkeypatch.setattr(connection, "resolve_project_connection_config", fail) monkeypatch.setattr(adapter_context, "resolve_adapter_connection_context", fail) exit_code: int = run_scope_command( request=ScopeCommandRequest(target="model:orders", json_output=True), @@ -100,7 +99,7 @@ def test_given_no_color_flag_when_rendering_text_to_color_terminal_then_output_i capsys: pytest.CaptureFixture[str], test_case: ScopeCommandCase, ) -> None: - monkeypatch.setattr(scope_command, "supports_color", lambda: True) + monkeypatch.setattr(color_support, "supports_color", lambda: True) index: ScopeIndex = report_scope_lookup().index outputs: list[str] = [] for no_color in (False, True): diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py index 2f8ccf6e4a..a45491a958 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py @@ -3,7 +3,7 @@ import re from pathlib import Path, PurePosixPath -from sqlbuild.cli.commands._helpers.skills.update import update_sqlbuild_skills +from sqlbuild._native import update_sqlbuild_skills from sqlbuild.cli.output.models import SkillUpdateResult diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py index 9444c490ca..c5fea0e828 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py @@ -5,11 +5,9 @@ import pytest -from sqlbuild.cli.commands._helpers.skills.update import ( - generated_marker, +from sqlbuild._native import ( load_packaged_skill_files, maintain_sqlbuild_skills, - skill_entry_file, update_sqlbuild_skills, ) from sqlbuild.cli.commands.exceptions import CliUserError @@ -29,6 +27,9 @@ write_project_files, ) +GENERATED_MARKER: str = "" +SKILL_ENTRY_FILE: str = "SKILL.md" + _SKILL_DIR: Path = Path(".agents/skills/sqlbuild") @@ -41,7 +42,7 @@ SkillDirectoryInstallTestCase( description="removes generated files that are no longer shipped", existing_files={ - _SKILL_DIR / "references/docs/retired_page.md": f"{generated_marker}\nold\n", + _SKILL_DIR / "references/docs/retired_page.md": f"{GENERATED_MARKER}\nold\n", }, expected_absent_paths=(_SKILL_DIR / "references/docs/retired_page.md",), ), @@ -79,7 +80,7 @@ def test_given_skill_directory_when_updating_then_it_matches_the_package( description="custom reference file blocks the whole install", existing_files={_SKILL_DIR / "references/testing.md": "custom testing notes\n"}, expected_error_fragment="references/testing.md", - expected_unwritten_path=_SKILL_DIR / skill_entry_file, + expected_unwritten_path=_SKILL_DIR / SKILL_ENTRY_FILE, ), ], ids=lambda case: case.description, @@ -131,7 +132,7 @@ def test_given_stale_reference_page_when_maintaining_then_directory_is_checked( ) skill_dir: Path = install_packaged_skill(project_dir=tmp_path, target="agents") stale_path: Path = skill_dir / test_case.stale_relative_path - stale_path.write_text(f"{generated_marker}\nold\n", encoding="utf-8") + stale_path.write_text(f"{GENERATED_MARKER}\nold\n", encoding="utf-8") result: SkillMaintenanceResult = maintain_sqlbuild_skills(project_dir=tmp_path) @@ -155,6 +156,7 @@ def test_given_stale_reference_page_when_maintaining_then_directory_is_checked( def test_given_packaged_skill_when_reading_then_entry_is_short_and_references_resolve( test_case: SkillContentTestCase, ) -> None: + packaged: dict[str, str] = load_packaged_skill_files() authored_paths: set[str] = set(packaged) - set(fnmatch.filter(packaged, "references/docs/*")) authored: dict[str, str] = {path: packaged[path] for path in authored_paths} @@ -169,7 +171,7 @@ def test_given_packaged_skill_when_reading_then_entry_is_short_and_references_re *(mentioned_sqb_commands(text=text) for text in authored.values()) ) - assert packaged[skill_entry_file].count("\n") <= test_case.max_entry_lines + assert packaged[SKILL_ENTRY_FILE].count("\n") <= test_case.max_entry_lines assert len({path.casefold() for path in packaged}) == len(packaged) assert links - packaged.keys() == test_case.expected_broken_links assert commands - {command.value for command in CliCommand} == ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py index 3bfb14db92..8bd8cee382 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py @@ -4,10 +4,8 @@ import pytest -from sqlbuild.cli.commands._helpers.skills.update import ( - ensure_generated_marker, - generated_marker, - load_packaged_skill_content, +from sqlbuild._native import ( + load_packaged_skill_files, maintain_sqlbuild_skills, update_sqlbuild_skills, ) @@ -28,6 +26,8 @@ write_project_files, ) +GENERATED_MARKER: str = "" + @pytest.mark.parametrize( "test_case", @@ -108,7 +108,7 @@ def test_given_skill_update_options_when_updating_then_writes_expected_targets( for expected_path in test_case.expected_written_paths: contents: str = read_relative_file(project_dir=tmp_path, relative_path=expected_path) assert contents.startswith("---\nname: sqlbuild\n") - assert generated_marker in contents + assert GENERATED_MARKER in contents assert test_case.expected_content_fragment in contents @@ -148,7 +148,7 @@ def test_given_global_skill_update_when_updating_then_writes_under_home_director for expected_path in test_case.expected_written_paths: contents: str = read_relative_file(project_dir=tmp_path, relative_path=expected_path) assert contents.startswith("---\nname: sqlbuild\n") - assert generated_marker in contents + assert GENERATED_MARKER in contents @pytest.mark.parametrize( @@ -175,29 +175,6 @@ def test_given_non_generated_skill_file_when_updating_without_force_then_raises_ assert test_case.expected_error_fragment in str(exc_info.value) -@pytest.mark.parametrize( - "test_case", - [ - SkillUpdateTestCase( - description="keeps skill frontmatter before generated marker", - expected_content_fragment=( - f"---\nname: sqlbuild\n---\n\n{generated_marker}\n# SQLBuild Skill" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_skill_frontmatter_when_adding_generated_marker_then_marker_follows_frontmatter( - test_case: SkillUpdateTestCase, -) -> None: - content: str = "---\nname: sqlbuild\n---\n# SQLBuild Skill\n" - - updated_content: str = ensure_generated_marker(content) - - assert updated_content.startswith("---\nname: sqlbuild\n") - assert test_case.expected_content_fragment in updated_content - - @pytest.mark.parametrize( "test_case", ( @@ -257,8 +234,8 @@ def test_given_project_skill_state_when_maintaining_then_reports_or_updates_owne assert test_case.expected_message_fragment in result.message for relative_path in test_case.expected_written_paths: - assert (tmp_path / relative_path).read_text(encoding="utf-8") == ensure_generated_marker( - load_packaged_skill_content() + assert (tmp_path / relative_path).read_text(encoding="utf-8") == ( + load_packaged_skill_files()["SKILL.md"] ) @@ -281,6 +258,7 @@ def test_given_custom_skill_collision_when_maintaining_then_file_is_not_overwrit test_case: SkillMaintenanceTestCase, tmp_path: Path, ) -> None: + prepare_skill_update_project( project_dir=tmp_path, project_config=test_case.project_config, diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py index a5f1fd369f..4be4a9b530 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py @@ -155,13 +155,6 @@ class RuntimeDiagnosticsTestCase: expected_absent_fragments: tuple[str, ...] -@dataclass(frozen=True) -class PlanningCompletionMessageTestCase: - description: str - message: str - expected_is_completion: bool - - @dataclass(frozen=True) class PlanningFinishTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py index 6186f3b380..f39b25c692 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.cli.progress.classes.connection_progress_reporter import ( +from sqlbuild._native import ( ConnectionProgressReporter, ) from sqlbuild.presentation.classes.cli_style import CliStyle diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py index 4b7457ac7e..0813bd636d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py @@ -6,13 +6,11 @@ import pytest -from sqlbuild.cli.progress.classes.planning_progress_reporter import ( +from sqlbuild._native import ( PlanningProgressReporter, - _is_planning_completion_message, ) from sqlbuild.presentation.classes.cli_style import CliStyle from tests.unit.src.sqlbuild.cli.commands.shared._helpers._test_types import ( - PlanningCompletionMessageTestCase, PlanningFinishTestCase, PlanningProgressTestCase, ) @@ -59,33 +57,6 @@ def test_given_planning_messages_when_reporting_then_writes_expected_output( assert stream.getvalue() == test_case.expected_output -@pytest.mark.parametrize( - "test_case", - [ - PlanningCompletionMessageTestCase( - description="treats applied clone plan as completion", - message="Applied clone plan. (0.10s)", - expected_is_completion=True, - ), - PlanningCompletionMessageTestCase( - description="treats finalized dbt run as completion", - message="Finalized dbt run.", - expected_is_completion=True, - ), - PlanningCompletionMessageTestCase( - description="treats evaluated rules as completion", - message="Evaluated rules. (built-in 0.00s, custom 0.00s)", - expected_is_completion=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_planning_message_when_classifying_completion_then_returns_expected_result( - test_case: PlanningCompletionMessageTestCase, -) -> None: - assert _is_planning_completion_message(test_case.message) == test_case.expected_is_completion - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 4ad8ceea79..8b48016772 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -9,6 +9,7 @@ import pytest +from sqlbuild._native import write_execution_header from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.cli.commands._helpers.test.sql_progress import ( @@ -25,7 +26,6 @@ from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( NestedCommandProgressCallbacks, ) -from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.cli.progress.models import AuditDisplayEntry, NestedProgressChildRow from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.planner.models import PlanOutput diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py index 3941dec374..a6be058340 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py @@ -3,20 +3,28 @@ from __future__ import annotations import json +from pathlib import Path from typing import cast from sqlbuild._native import ( + ExecutionEventWriter, + IntegrationResultEnvelope, + TerminalEventIndex, format_audit_execution_json, format_build_execution_json, format_test_execution_json, + terminal_event_index_scope, ) +from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome +from sqlbuild.runtime.observability.models import LifecycleEvent from tests.unit.src.sqlbuild.cli.output.main.plan.helpers import build_plan_output +from tests.unit.src.sqlbuild.runtime.observability.helpers import lifecycle_event def formatted_model_asset(result: ModelExecutionResult) -> dict[str, object]: @@ -54,3 +62,36 @@ def formatted_sql_test_step(step: StepResult) -> dict[str, object]: ) ) return cast(list[dict[str, object]], check["steps"])[0] + + +def written_integration_record( + *, result: object, terminal: LifecycleEvent, directory: Path, command: str = "build" +) -> IntegrationResultEnvelope: + """Write the integration record of one result after its terminal and decode it.""" + + path: Path = directory / "integration.jsonl" + index: TerminalEventIndex = TerminalEventIndex() + index.consume(terminal) + with terminal_event_index_scope(index): + writer: ExecutionEventWriter = ExecutionEventWriter(path=path) + writer.write_build_result(result=result, plan=None, command=command) + writer.close() + return IntegrationResultEnvelope.from_json(path.read_text(encoding="utf-8")) + + +def audit_terminal(result: AuditExecutionResult) -> LifecycleEvent: + """Return the completed terminal of one audit result.""" + + return lifecycle_event( + "resource_attempt_completed", + run_id="run-1", + resource_id=audit_resource_id( + audit_name=result.audit_name, + attachment_kind=result.attachment_kind, + attached_target_kind=result.attached_target_kind, + attached_target_name=result.attached_target_name, + attached_column_name=result.attached_column_name, + ), + resource_attempt_id="attempt-1", + payload={"resource_kind": "audit", "resource_name": result.audit_name, "attempt_number": 1}, + ) diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py index ce361fbc90..9502a17e50 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py @@ -2,10 +2,11 @@ from __future__ import annotations +from pathlib import Path + import pytest -from sqlbuild.cli.output._helpers.integration_result import _audit_result -from sqlbuild.cli.output.models import IntegrationCheckResult +from sqlbuild._native import IntegrationCheckResult from sqlbuild.compiler.auditing.models import ( MeasurementThresholdBound, MeasurementThresholds, @@ -21,6 +22,10 @@ from tests.unit.src.sqlbuild.cli.output._helpers._test_types import ( MeasurementAuditOutputTestCase, ) +from tests.unit.src.sqlbuild.cli.output._helpers.helpers import ( + audit_terminal, + written_integration_record, +) @pytest.mark.parametrize( @@ -35,7 +40,7 @@ ids=lambda case: case.description, ) def test_given_measurement_outcome_when_projecting_then_status_and_passed_remain_distinct( - test_case: MeasurementAuditOutputTestCase, + test_case: MeasurementAuditOutputTestCase, tmp_path: Path ) -> None: result: AuditExecutionResult = AuditExecutionResult( audit_name="valid_rate", @@ -49,7 +54,9 @@ def test_given_measurement_outcome_when_projecting_then_status_and_passed_remain evaluation_mode=AuditEvaluationMode.MEASUREMENT, ) - check: IntegrationCheckResult = _audit_result(result) + check: IntegrationCheckResult = written_integration_record( + result=result, terminal=audit_terminal(result), directory=tmp_path, command="audit" + ).checks[0] assert check.status == test_case.expected_status assert check.audit_definition_name == "dq_column_rate" @@ -63,7 +70,7 @@ def test_given_measurement_outcome_when_projecting_then_status_and_passed_remain ids=lambda case: case.description, ) def test_given_measurement_audit_when_projecting_then_all_summary_fields_are_retained( - test_case: MeasurementAuditOutputTestCase, + test_case: MeasurementAuditOutputTestCase, tmp_path: Path ) -> None: result: AuditExecutionResult = AuditExecutionResult( audit_name="valid_rate", @@ -86,7 +93,9 @@ def test_given_measurement_audit_when_projecting_then_all_summary_fields_are_ret evidence_truncated=True, ) - check: IntegrationCheckResult = _audit_result(result) + check: IntegrationCheckResult = written_integration_record( + result=result, terminal=audit_terminal(result), directory=tmp_path, command="audit" + ).checks[0] assert check.status == test_case.expected_status assert check.evaluation_mode == "measurement" @@ -108,7 +117,7 @@ def test_given_measurement_audit_when_projecting_then_all_summary_fields_are_ret ids=lambda case: case.description, ) def test_given_violations_audit_when_projecting_then_measurement_summary_is_absent( - test_case: MeasurementAuditOutputTestCase, + test_case: MeasurementAuditOutputTestCase, tmp_path: Path ) -> None: result: AuditExecutionResult = AuditExecutionResult( audit_name="not_null", @@ -120,7 +129,9 @@ def test_given_violations_audit_when_projecting_then_measurement_summary_is_abse executed_sql="SELECT 1 WHERE FALSE", ) - check: IntegrationCheckResult = _audit_result(result) + check: IntegrationCheckResult = written_integration_record( + result=result, terminal=audit_terminal(result), directory=tmp_path, command="audit" + ).checks[0] assert check.status == test_case.expected_status assert check.evaluation_mode == "violations" diff --git a/tests/unit/src/sqlbuild/cli/output/classes/helpers.py b/tests/unit/src/sqlbuild/cli/output/classes/helpers.py index 646345e35b..b37551bc3a 100644 --- a/tests/unit/src/sqlbuild/cli/output/classes/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/classes/helpers.py @@ -1,9 +1,10 @@ """Builders for terminal integration-result tests.""" +import json from collections.abc import Mapping from typing import cast -from sqlbuild.cli.output.models import IntegrationAssetResult +from sqlbuild._native import IntegrationAssetResult, IntegrationResultEnvelope from sqlbuild.runtime.observability.types import JSONValue @@ -56,12 +57,19 @@ def build_maximum_start_safety(action: str) -> Mapping[str, JSONValue]: ) +def decode_asset(asset: Mapping[str, object]) -> IntegrationAssetResult: + """Decode a valid record carrying the given asset and return its validated asset.""" + + payload: dict[str, object] = build_valid_integration_payload() + payload["asset"] = {"kind": "model", "name": "orders", "status": "success", **asset} + decoded: IntegrationAssetResult | None = IntegrationResultEnvelope.from_json( + json.dumps(payload) + ).asset + assert decoded is not None + return decoded + + def build_integration_asset_with_maximum_start(action: str) -> IntegrationAssetResult: - """Construct an asset carrying canonical maximum-start action metadata.""" + """Decode an asset carrying canonical maximum-start action metadata.""" - return IntegrationAssetResult( - kind="model", - name="orders", - status="success", - maximum_start_safety=build_maximum_start_safety(action), - ) + return decode_asset({"maximum_start_safety": build_maximum_start_safety(action)}) diff --git a/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py b/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py index 137f0e1da1..6177791a4d 100644 --- a/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py +++ b/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py @@ -8,26 +8,18 @@ from pathlib import Path from threading import Event, Thread from typing import Any, cast -from unittest.mock import Mock import pytest from sqlbuild._native import ( + ExecutionEventWriter, + IntegrationAssetResult, + IntegrationResultEnvelope, TerminalEventClaim, TerminalEventIndex, current_terminal_event_index, terminal_event_index_scope, ) -from sqlbuild.cli.output.classes import execution_event_writer -from sqlbuild.cli.output.classes.execution_event_writer import ExecutionEventWriter -from sqlbuild.cli.output.constants import ( - MAX_INTEGRATION_NESTING_DEPTH, - MAX_INTEGRATION_STRING_CHARS, -) -from sqlbuild.cli.output.models import ( - IntegrationAssetResult, - IntegrationResultEnvelope, -) from sqlbuild.executor.clone.types import CloneAction from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -46,9 +38,13 @@ from tests.unit.src.sqlbuild.cli.output.classes.helpers import ( build_integration_asset_with_maximum_start, build_valid_integration_payload, + decode_asset, ) from tests.unit.src.sqlbuild.runtime.observability.helpers import lifecycle_event +MAX_INTEGRATION_STRING_CHARS: int = 2_048 +MAX_INTEGRATION_NESTING_DEPTH: int = 6 + @pytest.mark.parametrize( "test_case", @@ -140,7 +136,7 @@ def test_given_unknown_check_severity_when_decoding_then_lists_allowed_values( @pytest.mark.parametrize( "test_case", ( - StructuralMetadataValidationTestCase("non-string key", {1: "value"}, "keys"), + StructuralMetadataValidationTestCase("non-string key", {1: "value"}, "unknown fields"), StructuralMetadataValidationTestCase("non-finite value", {"limit": float("inf")}, "finite"), StructuralMetadataValidationTestCase( "unknown key", {"user_metadata": "secret"}, "unknown fields" @@ -162,12 +158,7 @@ def test_given_unsafe_structural_metadata_when_constructing_asset_then_validatio test_case: StructuralMetadataValidationTestCase, ) -> None: with pytest.raises(ObservabilityValidationError, match=test_case.expected_error): - _ = IntegrationAssetResult( - kind="model", - name="orders", - status="success", - microbatch=cast(Mapping[str, Any], test_case.value), - ) + _ = decode_asset({"microbatch": cast(Mapping[str, Any], test_case.value)}) @pytest.mark.parametrize( @@ -181,9 +172,7 @@ def test_given_unsafe_structural_metadata_when_constructing_asset_then_validatio def test_given_canonical_clone_action_when_constructing_asset_then_integration_contract_accepts_it( test_case: IntegrationActionContractTestCase, ) -> None: - asset: IntegrationAssetResult = IntegrationAssetResult( - kind="model", name="orders", status="success", action=test_case.expected_action - ) + asset: IntegrationAssetResult = decode_asset({"action": test_case.expected_action}) assert asset.action == test_case.expected_action @@ -217,11 +206,8 @@ def test_given_canonical_maximum_start_action_when_constructing_asset_then_contr def test_given_canonical_microbatch_action_when_constructing_asset_then_contract_accepts_it( test_case: IntegrationActionContractTestCase, ) -> None: - asset: IntegrationAssetResult = IntegrationAssetResult( - kind="model", - name="orders", - status="success", - microbatch={"action": test_case.expected_action}, + asset: IntegrationAssetResult = decode_asset( + {"microbatch": {"action": test_case.expected_action}} ) assert asset.microbatch["action"] == test_case.expected_action @@ -443,7 +429,7 @@ def test_given_terminal_and_result_when_writing_then_canonical_envelope_is_flush ProjectionDegradationTestCase( description="successful execution with invalid optional projection", expected_warning=( - "warning: integration output degraded: injected projection failure; " + "warning: integration output degraded: integration staging_relation is invalid; " "warehouse execution completed successfully" ), expected_degraded=True, @@ -454,7 +440,6 @@ def test_given_terminal_and_result_when_writing_then_canonical_envelope_is_flush def test_given_successful_execution_when_integration_projection_fails_then_degraded_record_is_written( test_case: ProjectionDegradationTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: path: Path = tmp_path / "events.jsonl" @@ -468,16 +453,14 @@ def test_given_successful_execution_when_integration_projection_fails_then_degra payload={"resource_kind": "model", "resource_name": "orders", "attempt_number": 1}, ) ) - monkeypatch.setattr( - execution_event_writer, - "build_integration_result", - Mock(side_effect=ObservabilityValidationError("injected projection failure")), - ) - with terminal_event_index_scope(index): writer: ExecutionEventWriter = ExecutionEventWriter(path=path) writer.write_build_result( - result=ModelExecutionResult(model_name="orders", status=ExecutionStatus.SUCCESS), + result=ModelExecutionResult( + model_name="orders", + status=ExecutionStatus.SUCCESS, + staging_relation="x" * 513, + ), plan=None, ) writer.close() diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/helpers.py b/tests/unit/src/sqlbuild/cli/progress/classes/helpers.py index a25ec85cf2..2c45c59d05 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/helpers.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/helpers.py @@ -6,7 +6,7 @@ from io import StringIO from threading import Event, Lock, Thread, current_thread -from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector +from sqlbuild._native import NativeProgressProjector from sqlbuild.observability import ( EventDispatcher, OperationAttributes, diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py index 8c04877bf8..7cd228ed06 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py @@ -8,11 +8,11 @@ import pytest from sqlbuild._native import ( + NativeProgressProjector, TerminalEventIndex, terminal_event_index_scope, ) from sqlbuild.cli.commands.classes.build_progress_callbacks import BuildProgressCallbacks -from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py index 19187794ed..8206664aec 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py @@ -8,7 +8,7 @@ import pytest -from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector +from sqlbuild._native import NativeProgressProjector from sqlbuild.observability import ( EventDispatcher, LifecycleEvent, diff --git a/tests/unit/src/sqlbuild/cli/progress/main/test_expectation_detail.py b/tests/unit/src/sqlbuild/cli/progress/main/test_expectation_detail.py index 9eeb54bcf0..0b7ffcd887 100644 --- a/tests/unit/src/sqlbuild/cli/progress/main/test_expectation_detail.py +++ b/tests/unit/src/sqlbuild/cli/progress/main/test_expectation_detail.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.cli.progress.main._expectation_detail import format_expectation_detail +from sqlbuild._native import format_expectation_detail from sqlbuild.executor.testing.models import ( SqlTestColumnDifference, SqlTestDifferenceSample, diff --git a/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py b/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py index 5ae0078cb1..775b1d4844 100644 --- a/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py +++ b/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py @@ -5,9 +5,9 @@ import pytest from _pytest.capture import CaptureResult +from sqlbuild._native import VerboseInspectionWriter from sqlbuild.adapter.relations.main.record_inspection_query import record_inspection_query from sqlbuild.adapter.relations.models import InspectionQueryRecord -from sqlbuild.cli.progress.classes.verbose_inspection_writer import VerboseInspectionWriter from sqlbuild.cli.progress.main.verbose_inspection_output import verbose_inspection_output from tests.unit.src.sqlbuild.cli.progress.main.verbose_inspection_output._test_types import ( VerboseInspectionFormatTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py index 588c02cae4..0b23b9b933 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py @@ -6,6 +6,9 @@ import pytest +from sqlbuild._native import cursor_observed_partition as observed_partition +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render from sqlbuild.compiler.planner._helpers.resolve.cursor import ( apply_typed_cursor_replay_policy, compute_cursor_bounds, @@ -20,9 +23,6 @@ CursorType, CursorWatermarkMode, ) -from sqlbuild.cursor_algebra.main.observed_partition import observed_partition -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py index 6c60254f6b..55d33f98f9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py @@ -4,9 +4,9 @@ import pytest +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.planner._helpers.resolve.cursor import compute_cursor_bounds from sqlbuild.compiler.planner.models import CursorBounds, ModelCursorSnapshot -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorStartBoundsTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index 9d72d22f3f..64b45550b2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -27,7 +28,6 @@ PlanAction, PlanReason, ) -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ConfiguredMicrobatchPolicyTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index f793f116ce..d163308c0a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -28,7 +29,6 @@ WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import BackfillAction -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( AuthoritativeCursorOverrideTestCase, MicrobatchCursorEndPlanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py index d1a1f4d0ac..cf37383578 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py @@ -4,6 +4,8 @@ import pytest +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render from sqlbuild.compiler.planner._helpers.resolve.cursor import cursor_inputs_without_rows from sqlbuild.compiler.planner._helpers.warehouse import snapshot as snapshot_module from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( @@ -17,8 +19,6 @@ ) from sqlbuild.compiler.planner.models import EmptyCursorInputDecision, ModelCursorSnapshot from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorFetchFailureTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py index 3987a610cb..f85047d1e8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import cursor_parse as parse from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( _CursorModelInfo, @@ -16,7 +17,6 @@ ) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.main.parse import parse from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner._helpers.warehouse._test_types import ( ConcurrentCursorBoundsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py index 0c40948f10..e2c255b594 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py @@ -2,11 +2,11 @@ import pytest +from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive +from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.main.exclusive_to_inclusive import exclusive_to_inclusive -from sqlbuild.cursor_algebra.main.inclusive_to_exclusive import inclusive_to_exclusive -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( AdvanceCursorEndTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py index 9e60e99eee..12f9cc2855 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py @@ -4,12 +4,12 @@ import pytest +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.spec.contracts.models import FutureCursorsConfig from sqlbuild.spec.contracts.types import FutureCursorAction from tests.unit.src.sqlbuild.compiler.planner.main._test_types import FutureCursorSafetyTestCase diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py index 5fab0d264c..34b2183d86 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py @@ -5,6 +5,7 @@ import pytest +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.planner._helpers.resolve.cursor import compute_cursor_bounds from sqlbuild.compiler.planner.exceptions import MaximumAutomaticStartError from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety @@ -13,7 +14,6 @@ MaximumStartPolicyInputs, ModelCursorSnapshot, ) -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig from sqlbuild.spec.contracts.types import FutureCursorAction from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( diff --git a/tests/unit/src/sqlbuild/cost/test_store.py b/tests/unit/src/sqlbuild/cost/test_store.py index aaad8c161b..b7ee3061e6 100644 --- a/tests/unit/src/sqlbuild/cost/test_store.py +++ b/tests/unit/src/sqlbuild/cost/test_store.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.cost.classes.run_cost_store import RunCostStore +from sqlbuild._native import list_cost_runs, read_cost_run, resolve_cost_run, write_cost_run from sqlbuild.cost.exceptions import CostArtifactError from sqlbuild.cost.models import CostRunRecord from tests.unit.src.sqlbuild.cost._test_types import ( @@ -35,10 +35,10 @@ def test_given_cost_run_when_writing_then_record_round_trips_exact_decimals( completed_at=datetime(2026, 8, 23, tzinfo=UTC), ) - path: Path = RunCostStore.write(project_dir=tmp_path, record=record) + path: Path = write_cost_run(project_dir=tmp_path, record=record) assert path == (tmp_path / "target" / "executions" / test_case.expected_run_ids[0] / "run.json") - assert RunCostStore.read(project_dir=tmp_path, run_id=test_case.expected_run_ids[0]) == record + assert read_cost_run(project_dir=tmp_path, run_id=test_case.expected_run_ids[0]) == record assert not (path.parent / ".run.json.tmp").exists() @@ -60,15 +60,15 @@ def test_given_multiple_cost_runs_when_listing_then_latest_completion_is_first( latest: CostRunRecord = build_cost_run_record( run_id="run-new", completed_at=origin + timedelta(minutes=1) ) - RunCostStore.write(project_dir=tmp_path, record=older) - RunCostStore.write(project_dir=tmp_path, record=latest) + write_cost_run(project_dir=tmp_path, record=older) + write_cost_run(project_dir=tmp_path, record=latest) - records: tuple[CostRunRecord, ...] = RunCostStore.list(project_dir=tmp_path) + records: tuple[CostRunRecord, ...] = list_cost_runs(project_dir=tmp_path) assert tuple(record.run_id for record in records) == test_case.expected_run_ids assert records == (latest, older) - assert RunCostStore.resolve(project_dir=tmp_path, selector="latest") == latest - assert RunCostStore.resolve(project_dir=tmp_path, selector="run-o") == older + assert resolve_cost_run(project_dir=tmp_path, selector="latest") == latest + assert resolve_cost_run(project_dir=tmp_path, selector="run-o") == older @pytest.mark.parametrize( @@ -94,11 +94,11 @@ def test_given_invalid_artifact_when_listing_then_valid_history_remains_availabl tmp_path: Path, ) -> None: completed_at: datetime = datetime(2026, 8, 23, tzinfo=UTC) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record(run_id="run-valid", completed_at=completed_at), ) - invalid_path: Path = RunCostStore.write( + invalid_path: Path = write_cost_run( project_dir=tmp_path, record=build_cost_run_record(run_id="run-invalid", completed_at=completed_at), ) @@ -111,7 +111,7 @@ def test_given_invalid_artifact_when_listing_then_valid_history_remains_availabl encoding="utf-8", ) - records: tuple[CostRunRecord, ...] = RunCostStore.list(project_dir=tmp_path) + records: tuple[CostRunRecord, ...] = list_cost_runs(project_dir=tmp_path) assert tuple(record.run_id for record in records) == test_case.expected_run_ids @@ -137,7 +137,7 @@ def test_given_unsafe_run_id_when_reading_then_path_escape_is_rejected( tmp_path: Path, ) -> None: with pytest.raises(CostArtifactError, match=test_case.expected_error_fragment): - RunCostStore.read(project_dir=tmp_path, run_id=test_case.run_id) + read_cost_run(project_dir=tmp_path, run_id=test_case.run_id) @pytest.mark.parametrize( @@ -155,11 +155,11 @@ def test_given_ambiguous_run_prefix_when_resolving_then_none_is_returned( test_case: CostStoreTestCase, tmp_path: Path ) -> None: origin: datetime = datetime(2026, 8, 23, tzinfo=UTC) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record(run_id=test_case.expected_run_ids[0], completed_at=origin), ) - RunCostStore.write( + write_cost_run( project_dir=tmp_path, record=build_cost_run_record( run_id=test_case.expected_run_ids[1], @@ -167,7 +167,7 @@ def test_given_ambiguous_run_prefix_when_resolving_then_none_is_returned( ), ) - resolved: CostRunRecord | None = RunCostStore.resolve(project_dir=tmp_path, selector="run-") + resolved: CostRunRecord | None = resolve_cost_run(project_dir=tmp_path, selector="run-") assert (resolved is not None) is test_case.expected_resolved @@ -193,10 +193,10 @@ def test_given_newer_running_record_when_resolving_latest_then_terminal_run_is_r build_cost_run_record(run_id="run-running", completed_at=origin + timedelta(minutes=1)), build_status="running", ) - RunCostStore.write(project_dir=tmp_path, record=completed) - RunCostStore.write(project_dir=tmp_path, record=running) + write_cost_run(project_dir=tmp_path, record=completed) + write_cost_run(project_dir=tmp_path, record=running) - resolved: CostRunRecord | None = RunCostStore.resolve(project_dir=tmp_path, selector="latest") + resolved: CostRunRecord | None = resolve_cost_run(project_dir=tmp_path, selector="latest") assert resolved == completed @@ -222,13 +222,13 @@ def test_given_malformed_decimal_artifact_when_listing_then_valid_runs_remain_av invalid: CostRunRecord = build_cost_run_record( run_id="run-invalid", completed_at=origin + timedelta(minutes=1) ) - RunCostStore.write(project_dir=tmp_path, record=valid) - invalid_path: Path = RunCostStore.write(project_dir=tmp_path, record=invalid) + write_cost_run(project_dir=tmp_path, record=valid) + invalid_path: Path = write_cost_run(project_dir=tmp_path, record=invalid) invalid_path.write_text( invalid_path.read_text(encoding="utf-8").replace('"3.00"', '"not-a-decimal"', 1), encoding="utf-8", ) - records: tuple[CostRunRecord, ...] = RunCostStore.list(project_dir=tmp_path) + records: tuple[CostRunRecord, ...] = list_cost_runs(project_dir=tmp_path) assert tuple(record.run_id for record in records) == test_case.expected_run_ids diff --git a/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py b/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py index 8a1ae8588f..9968b12aef 100644 --- a/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py +++ b/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py @@ -4,27 +4,27 @@ import pytest -from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.main.cap_from_end import cap_from_end -from sqlbuild.cursor_algebra.main.cap_from_start import cap_from_start -from sqlbuild.cursor_algebra.main.clamp import clamp -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.exclusive_end_from_observed_max import ( - exclusive_end_from_observed_max, +from sqlbuild._native import cursor_cap_from_end as cap_from_end +from sqlbuild._native import cursor_cap_from_start as cap_from_start +from sqlbuild._native import cursor_clamp as clamp +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import ( + cursor_exclusive_end_from_observed_max as exclusive_end_from_observed_max, ) -from sqlbuild.cursor_algebra.main.exclusive_to_inclusive import exclusive_to_inclusive -from sqlbuild.cursor_algebra.main.floor_to_grain import floor_to_grain -from sqlbuild.cursor_algebra.main.inclusive_to_exclusive import inclusive_to_exclusive -from sqlbuild.cursor_algebra.main.max_bound import max_bound -from sqlbuild.cursor_algebra.main.merge import merge -from sqlbuild.cursor_algebra.main.min_bound import min_bound -from sqlbuild.cursor_algebra.main.next_boundary import next_boundary -from sqlbuild.cursor_algebra.main.observed_partition import observed_partition -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render -from sqlbuild.cursor_algebra.main.sentinel_from_token import sentinel_from_token -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token -from sqlbuild.cursor_algebra.main.split_into_batches import split_into_batches +from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive +from sqlbuild._native import cursor_floor_to_grain as floor_to_grain +from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive +from sqlbuild._native import cursor_max_bound as max_bound +from sqlbuild._native import cursor_merge as merge +from sqlbuild._native import cursor_min_bound as min_bound +from sqlbuild._native import cursor_next_boundary as next_boundary +from sqlbuild._native import cursor_observed_partition as observed_partition +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render +from sqlbuild._native import cursor_sentinel_from_token as sentinel_from_token +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import cursor_split_into_batches as split_into_batches +from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.cursor_algebra.models import AlignedInterval, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from tests.unit.src.sqlbuild.cursor_algebra.main._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index 68564a0d8f..bc41d529fa 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -9,9 +9,9 @@ import pytest +from sqlbuild._native import NativeProgressProjector from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import ( diff --git a/tests/unit/src/sqlbuild/executor/load/test_cursors.py b/tests/unit/src/sqlbuild/executor/load/test_cursors.py index 6563768eb7..12e9540275 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_cursors.py +++ b/tests/unit/src/sqlbuild/executor/load/test_cursors.py @@ -7,9 +7,9 @@ import pytest +from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/tests/unit/src/sqlbuild/executor/load/test_run.py b/tests/unit/src/sqlbuild/executor/load/test_run.py index 57f4c2b1cd..8f75b2c567 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_run.py +++ b/tests/unit/src/sqlbuild/executor/load/test_run.py @@ -8,8 +8,8 @@ import pytest +from sqlbuild._native import NativeProgressProjector from sqlbuild.cli.commands.classes.load_progress_reporter import LoadProgressReporter -from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import SkipMode diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py index 69e1e25b51..523b555642 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py @@ -9,11 +9,11 @@ import pytest from sqlbuild._native import ( + ExecutionEventWriter, TerminalEventIndex, terminal_event_index_scope, ) from sqlbuild.cli.commands._helpers.check.core import format_check_json -from sqlbuild.cli.output.classes.execution_event_writer import ExecutionEventWriter from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index 4bca9ee616..cc01478c10 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -8,6 +8,8 @@ import duckdb import pytest +from sqlbuild._native import cursor_render as render +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation @@ -28,8 +30,6 @@ PlanAction, PlanReason, ) -from sqlbuild.cursor_algebra.main.render import render -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py index 48a4adae30..4d865e97ca 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py @@ -9,11 +9,11 @@ import pytest +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.main.compare import compare -from sqlbuild.cursor_algebra.main.inclusive_to_exclusive import inclusive_to_exclusive -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.executor.run._helpers.materializations.microbatch import compute_batch_windows from sqlbuild.executor.run.models import BatchWindow diff --git a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py index ade0b9641a..4da70ff215 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py @@ -6,9 +6,7 @@ from pathlib import Path from typing import Any -from sqlbuild.cli.output.models import IntegrationCheckResult, IntegrationResultEnvelope -from sqlbuild.cli.output.types import IntegrationOutputKind -from sqlbuild.compiler.auditing.types import AuditSeverity +from sqlbuild._native import IntegrationResultEnvelope def assert_exhaustive_enum_partition[EnumT: StrEnum]( @@ -61,46 +59,57 @@ def integration_result_payload( def build_check_integration_envelope( - *, check_id: str, name: str, event_id: str, attempt_id: str, event_sequence: int + *, + check_id: str, + name: str, + event_id: str, + attempt_id: str, + event_sequence: int, + command: str = "audit", + check_overrides: Mapping[str, object] | None = None, ) -> IntegrationResultEnvelope: - return IntegrationResultEnvelope( - schema_version=1, - record_kind="integration_result", - event_id=event_id, - event_sequence=event_sequence, - event_type="resource_attempt_completed", - occurred_at="2026-09-02T12:00:00+00:00", - invocation_id="invocation-1", - run_id="run-1", - resource_id=check_id, - resource_attempt_id=attempt_id, - operation_id=None, - statement_id=None, - resource_kind="audit", - resource_name=name, - attempt_number=1, - duration_ms=1, - output_kind=IntegrationOutputKind.CHECK, - command="audit", - checks=( - IntegrationCheckResult( - kind="audit", - audit_definition_name="test_audit", - name=name, - check_id=check_id, - dag_check_id=check_id, - passed=True, - status="pass", - severity=AuditSeverity.ERROR, - asset_name="orders", - attachment_kind="model", - attached_column_name="order_id", - attached_target_name="orders", - run_scope_phase="final", - row_count=0, - reused=False, - ), - ), + check: dict[str, object] = { + "kind": "audit", + "audit_definition_name": "test_audit", + "name": name, + "check_id": check_id, + "dag_check_id": check_id, + "passed": True, + "status": "pass", + "severity": "error", + "asset_name": "orders", + "attachment_kind": "model", + "attached_column_name": "order_id", + "attached_target_name": "orders", + "run_scope_phase": "final", + "row_count": 0, + "reused": False, + **(check_overrides or {}), + } + return IntegrationResultEnvelope.from_json( + json.dumps( + { + "schema_version": 1, + "record_kind": "integration_result", + "event_id": event_id, + "event_sequence": event_sequence, + "event_type": "resource_attempt_completed", + "occurred_at": "2026-09-02T12:00:00+00:00", + "invocation_id": "invocation-1", + "run_id": "run-1", + "resource_id": check_id, + "resource_attempt_id": attempt_id, + "operation_id": None, + "statement_id": None, + "resource_kind": "audit", + "resource_name": name, + "attempt_number": 1, + "duration_ms": 1, + "output_kind": "check", + "command": command, + "checks": [check], + } + ) ) @@ -442,7 +451,7 @@ def write_blocking_audit_event_command(*, root: Path, release_path: Path) -> lis attempt_id="attempt-live-audit", event_sequence=1, ) - event_payload: str = envelope.to_json() + "\n" + event_payload: str = envelope.to_json() execution_payload: str = json.dumps( { "version": 1, diff --git a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py index b088b56696..a80d262c6a 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py @@ -4,7 +4,6 @@ from collections import defaultdict from collections.abc import Generator, Iterator, Mapping from concurrent.futures import Future, ThreadPoolExecutor -from dataclasses import replace from pathlib import Path from threading import Event from typing import Any, cast @@ -13,8 +12,7 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild._native import serialize_maximum_start_safety -from sqlbuild.cli.output.models import IntegrationCheckResult, IntegrationResultEnvelope +from sqlbuild._native import IntegrationResultEnvelope, serialize_maximum_start_safety from sqlbuild.compiler.planner.models import MaximumStartSafetyEvidence from sqlbuild.integrations.dagster import ( SqlBuildCliResource, @@ -305,14 +303,12 @@ def test_given_two_audit_envelopes_when_projecting_then_each_check_keeps_own_can def test_given_standalone_check_command_when_projecting_live_result_then_parent_need_not_materialize( test_case: DagsterStandaloneCheckCommandTestCase, ) -> None: - envelope: IntegrationResultEnvelope = replace( - build_check_integration_envelope( - check_id="audit:not_null:model:orders:order_id", - name="not_null", - event_id="event-live-check", - attempt_id="attempt-live-check", - event_sequence=1, - ), + envelope: IntegrationResultEnvelope = build_check_integration_envelope( + check_id="audit:not_null:model:orders:order_id", + name="not_null", + event_id="event-live-check", + attempt_id="attempt-live-check", + event_sequence=1, command=test_case.command, ) @@ -378,14 +374,12 @@ def test_given_measurement_check_when_projecting_then_dagster_metadata_and_warni event_id="event-measurement", attempt_id="attempt-measurement", event_sequence=5, + check_overrides={ + "status": test_case.expected_status, + "passed": test_case.expected_passed, + **test_case.expected_metadata, + }, ) - check: IntegrationCheckResult = replace( - envelope.checks[0], - status=test_case.expected_status, - passed=test_case.expected_passed, - **test_case.expected_metadata, - ) - envelope = replace(envelope, checks=(check,)) results: tuple[Any, ...] = _build_results_from_integration_result( dg=dg, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py index 8a44dae9ea..760401f6ca 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild.lint._helpers.headers import scan_headers, sql_body_ranges +from sqlbuild._native import scan_headers, sql_body_ranges from sqlbuild.lint.models import HeaderSpan, LintBody from sqlbuild.lint.types import NativeLintPreparationRequest, NativePreparedSql diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py b/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py index 3ec89456f7..126ad59e2f 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.lint._helpers.native import format_native_headers +from sqlbuild._native import format_native_headers from sqlbuild.lint.models import LintConfig from tests.unit.src.sqlbuild.lint._helpers._test_types import HeaderAuditWrapTestCase diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py b/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py index 44546af55e..8d6f25e05c 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.lint._helpers.headers import scan_headers, sql_body_ranges +from sqlbuild._native import scan_headers, sql_body_ranges from tests.unit.src.sqlbuild.lint._helpers._test_types import ( HeaderSpanTextTestCase, ScanHeadersTestCase, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_native.py b/tests/unit/src/sqlbuild/lint/_helpers/test_native.py index 3c66ffd29c..2f5a603626 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_native.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_native.py @@ -6,9 +6,8 @@ import pytest +from sqlbuild._native import format_native_headers, lint_native_headers, scan_headers from sqlbuild.compiler.discovery.exceptions import SqlScenarioParseError -from sqlbuild.lint._helpers.headers import scan_headers -from sqlbuild.lint._helpers.native import format_native_headers, lint_native_headers from sqlbuild.lint.models import LintConfig from tests.unit.src.sqlbuild.compiler.discovery.helpers import ( parse_model_sql, diff --git a/tests/unit/src/sqlbuild/microbatches/test_sql.py b/tests/unit/src/sqlbuild/microbatches/test_sql.py index 7acacadbca..c0a21be303 100644 --- a/tests/unit/src/sqlbuild/microbatches/test_sql.py +++ b/tests/unit/src/sqlbuild/microbatches/test_sql.py @@ -6,6 +6,8 @@ import pytest +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter @@ -14,8 +16,6 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.main.parse import parse -from sqlbuild.cursor_algebra.main.render import render from sqlbuild.microbatches._helpers.sql import ( build_create_table_sql, build_insert_many_sql, diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py b/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py index 8b9d0a8d65..16e50370c2 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.presentation.classes.cli_document import CliDocument +from sqlbuild._native import CliDocument from sqlbuild.presentation.classes.cli_style import CliStyle from tests.unit.src.sqlbuild.presentation.classes._test_types import CliDocumentTestCase diff --git a/tests/unit/src/sqlbuild/presentation/main/test_count_header.py b/tests/unit/src/sqlbuild/presentation/main/test_count_header.py index 4b73b9ace4..b3dde32fbe 100644 --- a/tests/unit/src/sqlbuild/presentation/main/test_count_header.py +++ b/tests/unit/src/sqlbuild/presentation/main/test_count_header.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import count_header_style from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.count_header import count_header_style from tests.unit.src.sqlbuild.presentation.main._test_types import CountHeaderTestCase diff --git a/tests/unit/src/sqlbuild/presentation/main/test_count_noun.py b/tests/unit/src/sqlbuild/presentation/main/test_count_noun.py index 31a49911d3..7fae72a03f 100644 --- a/tests/unit/src/sqlbuild/presentation/main/test_count_noun.py +++ b/tests/unit/src/sqlbuild/presentation/main/test_count_noun.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.presentation.main.count_noun import format_count_noun +from sqlbuild._native import format_count_noun from tests.unit.src.sqlbuild.presentation.main._test_types import CountNounTestCase diff --git a/tests/unit/src/sqlbuild/presentation/main/test_summary_footer.py b/tests/unit/src/sqlbuild/presentation/main/test_summary_footer.py index 4b94aa2d14..36736d962c 100644 --- a/tests/unit/src/sqlbuild/presentation/main/test_summary_footer.py +++ b/tests/unit/src/sqlbuild/presentation/main/test_summary_footer.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.presentation.main.summary_footer import format_summary_footer +from sqlbuild._native import format_summary_footer from tests.unit.src.sqlbuild.presentation.main._test_types import SummaryFooterTestCase diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py index bdded55451..65d1375bf6 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py @@ -4,13 +4,12 @@ import pytest -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue -from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( - _RuleTestEvidence, +from sqlbuild._native import ( custom_rule_implementation_fingerprint, custom_rule_test_evidence, ) +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult @@ -256,13 +255,13 @@ def test_given_rule_and_test_source_when_collecting_evidence_then_only_exact_sta test_path.write_text(test_case.test_source, encoding="utf-8") rule: Rule = load_custom_rule(root=tmp_path, configured_path=configured_path) - evidence: tuple[_RuleTestEvidence, ...] = custom_rule_test_evidence( + evidence: list[tuple[str, int, int, str]] = custom_rule_test_evidence( rule=rule, project_dir=tmp_path ) assert len(evidence) == test_case.expected_count - assert all(item.owner == "tests/test_custom.py:test_exact" for item in evidence) - assert all(item.line > 0 and item.column > 0 for item in evidence) + assert all(owner == "tests/test_custom.py:test_exact" for _, _, _, owner in evidence) + assert all(line > 0 and column > 0 for _, line, column, _ in evidence) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/spec/contracts/_helpers/test_config_values.py b/tests/unit/src/sqlbuild/spec/contracts/_helpers/test_config_values.py index 36a080b364..54e13cee3f 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/_helpers/test_config_values.py +++ b/tests/unit/src/sqlbuild/spec/contracts/_helpers/test_config_values.py @@ -2,12 +2,14 @@ import pytest +from sqlbuild._native import ( + get_config_bool, + get_config_cursor_bound, + get_config_int, + get_config_str, + get_config_string_tuple, +) from sqlbuild.spec.contracts.exceptions import ConfigValueTypeError -from sqlbuild.spec.contracts.main.get_config_bool import get_config_bool -from sqlbuild.spec.contracts.main.get_config_cursor_bound import get_config_cursor_bound -from sqlbuild.spec.contracts.main.get_config_int import get_config_int -from sqlbuild.spec.contracts.main.get_config_str import get_config_str -from sqlbuild.spec.contracts.main.get_config_string_tuple import get_config_string_tuple from tests.unit.src.sqlbuild.spec.contracts._helpers._test_types import ( ConfigValueErrorTestCase, ConfigValueSuccessTestCase,