diff --git a/Makefile b/Makefile index 5e1dfc37ec..3f16e3e5ad 100644 --- a/Makefile +++ b/Makefile @@ -139,6 +139,7 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/playground \ tests/e2e/src/sqlbuild/cli/commands/main/postgres \ tests/e2e/src/sqlbuild/cli/commands/main/query \ + tests/e2e/src/sqlbuild/cli/commands/main/refactor \ tests/e2e/src/sqlbuild/cli/commands/main/scope \ tests/e2e/src/sqlbuild/cli/commands/main/seed \ tests/e2e/src/sqlbuild/cli/commands/main/skills \ diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 186c8ab787..a61408c6cc 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -154,6 +154,12 @@ fn analyze_sql_uses_json(py: Python<'_>, request_json: &str) -> PyResult .map_err(value_error) } +#[pyfunction] +fn analyze_column_references_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| crate::column_references::main::analyze::analyze_json(request_json)) + .map_err(value_error) +} + #[pyfunction] fn analyze_queries_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| crate::query_analysis::main::analyze::analyze_json(request_json)) @@ -429,6 +435,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(schema_validation_json, module)?)?; module.add_function(wrap_pyfunction!(schema_validations_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_sql_uses_json, module)?)?; + module.add_function(wrap_pyfunction!(analyze_column_references_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_queries_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_project_queries_json, module)?)?; module.add_function(wrap_pyfunction!( diff --git a/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs b/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs new file mode 100644 index 0000000000..392ffeb864 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs @@ -0,0 +1,592 @@ +//! Resolve every reference to one relation column, with the spans a rename rewrites. + +use polyglot_sql::{ + Dialect, DialectType, Expression, MappingSchema, Resolver, Scope, ValidationSchema, + build_scope, mapping_schema_from_validation_schema_with_dialect, walk_in_scope, +}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; + +#[derive(Deserialize)] +struct ReferenceRequest { + sql: String, + #[serde(default = "default_dialect")] + dialect: String, + schema: ValidationSchema, + column: String, + #[serde(default)] + target_tables: Vec, + #[serde(default)] + target_ctes: Vec, + #[serde(default)] + output_ctes: Vec, +} + +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct ReferenceResponse { + parsed: bool, + references: Vec, + stars: Vec, + joins: Vec, + unresolved: Vec, + outputs: Vec, + star_outputs: Vec, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct ReferenceSite { + start: usize, + end: usize, + name_start: usize, + name_end: usize, + scope: String, + projection: Option, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct ProjectionSite { + alias: Option, + alias_start: Option, + alias_end: Option, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct StarSite { + start: Option, + end: Option, + scope: String, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct SpanSite { + start: Option, + end: Option, + scope: String, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct OutputSite { + scope: String, + start: Option, + end: Option, + alias_start: Option, + alias_end: Option, +} + +const ROOT_SCOPE: &str = "root"; +const DERIVED_SCOPE: &str = "derived"; +const SUBQUERY_SCOPE: &str = "subquery"; +const CTE_SCOPE_PREFIX: &str = "cte:"; +const STAR: &str = "*"; + +struct Analysis<'a> { + schema: &'a MappingSchema, + column: String, + target_tables: HashSet, + target_ctes: HashSet, + output_ctes: HashSet, + response: ReferenceResponse, + seen: HashSet<(usize, usize)>, + later_branch: bool, +} + +fn default_dialect() -> String { + "generic".to_string() +} + +pub(crate) fn analyze_json(request_json: &str) -> Result { + let request: ReferenceRequest = + serde_json::from_str(request_json).map_err(|error| error.to_string())?; + let dialect: DialectType = request.dialect.parse().map_err(|error| { + format!( + "unsupported Polyglot dialect '{}': {error}", + request.dialect + ) + })?; + let Ok(statements) = Dialect::get(dialect).parse(&request.sql) else { + return serde_json::to_string(&ReferenceResponse::default()) + .map_err(|error| error.to_string()); + }; + let schema = mapping_schema_from_validation_schema_with_dialect(&request.schema, dialect); + let scopes: Vec = statements.iter().map(build_scope).collect(); + let ctes: Vec<&polyglot_sql::expressions::Cte> = scopes.iter().flat_map(scope_ctes).collect(); + let target_ctes: HashSet = star_derived_ctes( + &ctes, + &lowered(&request.target_tables), + lowered(&request.target_ctes), + ); + let mut analysis = Analysis { + schema: &schema, + column: request.column.to_lowercase(), + target_tables: lowered(&request.target_tables), + target_ctes, + output_ctes: lowered(&request.output_ctes), + response: ReferenceResponse { + parsed: true, + ..ReferenceResponse::default() + }, + seen: HashSet::new(), + later_branch: false, + }; + for scope in &scopes { + analysis.visit(scope, ROOT_SCOPE.to_string(), &[], false); + } + serde_json::to_string(&analysis.response).map_err(|error| error.to_string()) +} + +fn lowered(names: &[String]) -> HashSet { + names.iter().map(|name| name.to_lowercase()).collect() +} + +fn scope_ctes(scope: &Scope) -> Vec<&polyglot_sql::expressions::Cte> { + let mut ctes: Vec<&polyglot_sql::expressions::Cte> = Vec::new(); + if let Expression::Cte(cte) = &scope.expression { + ctes.push(cte); + } + for child in scope + .cte_scopes + .iter() + .chain(scope.derived_table_scopes.iter()) + .chain(scope.subquery_scopes.iter()) + .chain(scope.udtf_scopes.iter()) + .chain(scope.union_scopes.iter()) + { + ctes.extend(scope_ctes(child)); + } + ctes +} + +/// CTEs whose output passes the target column through a star become targets themselves. +fn star_derived_ctes( + ctes: &[&polyglot_sql::expressions::Cte], + target_tables: &HashSet, + seed: HashSet, +) -> HashSet { + let mut derived: HashSet = seed; + loop { + let before = derived.len(); + for cte in ctes { + let name = cte.alias.name.to_lowercase(); + if !derived.contains(&name) + && selects_star_over_targets(&cte.this, target_tables, &derived) + { + derived.insert(name); + } + } + if derived.len() == before { + return derived; + } + } +} + +/// Only the first branch of a set operation names its output columns. +fn selects_star_over_targets( + body: &Expression, + target_tables: &HashSet, + target_ctes: &HashSet, +) -> bool { + first_branch_select(body) + .is_some_and(|select| select_stars_over_targets(select, target_tables, target_ctes)) +} + +fn select_stars_over_targets( + select: &polyglot_sql::expressions::Select, + target_tables: &HashSet, + target_ctes: &HashSet, +) -> bool { + let relations: HashMap = select_relation_names(select); + for item in &select.expressions { + let covered: bool = match star_table(item) { + Some(None) => relations + .values() + .any(|name| is_target_name(name, target_tables, target_ctes)), + Some(Some(qualifier)) => relations + .get(&qualifier) + .is_some_and(|name| is_target_name(name, target_tables, target_ctes)), + None => false, + }; + if covered { + return true; + } + } + false +} + +fn is_target_name( + name: &str, + target_tables: &HashSet, + target_ctes: &HashSet, +) -> bool { + target_tables.contains(name) || target_ctes.contains(name) +} + +fn first_branch_select(body: &Expression) -> Option<&polyglot_sql::expressions::Select> { + match body { + Expression::Select(select) => Some(select), + Expression::Union(union) => first_branch_select(&union.left), + Expression::Intersect(intersect) => first_branch_select(&intersect.left), + Expression::Except(except) => first_branch_select(&except.left), + Expression::Subquery(subquery) => first_branch_select(&subquery.this), + Expression::Paren(paren) => first_branch_select(&paren.this), + _ => None, + } +} + +/// Map each visible alias of a SELECT's FROM and JOIN tables to its lower-cased table name. +fn select_relation_names(select: &polyglot_sql::expressions::Select) -> HashMap { + let mut names: HashMap = HashMap::new(); + let tables = select + .from + .iter() + .flat_map(|from| from.expressions.iter()) + .chain(select.joins.iter().map(|join| &join.this)); + for table in tables { + if let Expression::Table(table) = table { + let name = table.name.name.to_lowercase(); + let visible = table + .alias + .as_ref() + .map_or_else(|| name.clone(), |alias| alias.name.to_lowercase()); + names.insert(visible, name); + } + } + names +} + +/// `Some(None)` for `*`, `Some(Some(qualifier))` for `t.*`, `None` otherwise. +fn star_table(item: &Expression) -> Option> { + match item { + Expression::Star(star) => Some(star.table.as_ref().map(|table| table.name.to_lowercase())), + Expression::Column(column) if column.name.name == STAR => { + Some(column.table.as_ref().map(|table| table.name.to_lowercase())) + } + _ => None, + } +} + +fn star_span(item: &Expression) -> (Option, Option) { + match item { + Expression::Star(star) => ( + star.span.map(|span| span.start), + star.span.map(|span| span.end), + ), + Expression::Column(column) => ( + column.span.map(|span| span.start), + column.span.map(|span| span.end), + ), + _ => (None, None), + } +} + +fn scope_select(expression: &Expression) -> Option<&polyglot_sql::expressions::Select> { + match select_body(expression)? { + Expression::Select(select) => Some(select), + _ => None, + } +} + +fn select_body(expression: &Expression) -> Option<&Expression> { + match expression { + Expression::Select(_) => Some(expression), + Expression::Cte(cte) => select_body(&cte.this), + Expression::Subquery(subquery) => select_body(&subquery.this), + Expression::Paren(paren) => select_body(&paren.this), + _ => None, + } +} + +impl Analysis<'_> { + /// Visit one scope; `later_branch` marks a set-operation branch that names no outputs. + fn visit(&mut self, scope: &Scope, role: String, outer: &[&Scope], later_branch: bool) { + if let Some(select) = scope_select(&scope.expression) { + let local: Scope = local_scope(scope, select); + let mut chain: Vec<&Scope> = vec![&local]; + chain.extend(outer.iter().copied()); + self.later_branch = later_branch; + self.visit_select(select, &scope.expression, &chain, &role); + } + for child in &scope.cte_scopes { + let name = match &child.expression { + Expression::Cte(cte) => cte.alias.name.clone(), + _ => String::new(), + }; + self.visit(child, format!("{CTE_SCOPE_PREFIX}{name}"), &[], false); + } + for child in &scope.derived_table_scopes { + self.visit(child, DERIVED_SCOPE.to_string(), &[], false); + } + let mut nested: Vec<&Scope> = vec![scope]; + nested.extend(outer.iter().copied()); + for child in scope.subquery_scopes.iter().chain(scope.udtf_scopes.iter()) { + self.visit(child, SUBQUERY_SCOPE.to_string(), &nested, false); + } + for (position, child) in scope.union_scopes.iter().enumerate() { + self.visit(child, role.clone(), outer, later_branch || position > 0); + } + } + + fn visit_select( + &mut self, + select: &polyglot_sql::expressions::Select, + expression: &Expression, + chain: &[&Scope], + role: &str, + ) { + self.record_outputs(select, role); + if !self.later_branch { + self.record_stars(select, chain[0], role); + } + self.record_joins(select, chain[0], role); + let Some(body) = select_body(expression) else { + return; + }; + for node in walk_in_scope(body, false) { + let Expression::Column(column) = node else { + continue; + }; + if column.name.name == STAR || column.name.name.to_lowercase() != self.column { + continue; + } + self.record_column(column, select, chain, role); + } + } + + fn record_column( + &mut self, + column: &polyglot_sql::expressions::Column, + select: &polyglot_sql::expressions::Select, + chain: &[&Scope], + role: &str, + ) { + let (Some(span), Some(name_span)) = (column.span, column.name.span) else { + return; + }; + if !self.seen.insert((span.start, span.end)) { + return; + } + match self.resolve(column, chain) { + Resolution::Target => self.response.references.push(ReferenceSite { + start: span.start, + end: span.end, + name_start: name_span.start, + name_end: name_span.end, + scope: role.to_string(), + projection: if self.later_branch { + None + } else { + projection_site(select, span.start) + }, + }), + Resolution::Other => {} + Resolution::Unknown => { + if chain.iter().any(|scope| self.has_target_source(scope)) { + self.response.unresolved.push(SpanSite { + start: Some(span.start), + end: Some(span.end), + scope: role.to_string(), + }); + } + } + } + } + + fn resolve(&self, column: &polyglot_sql::expressions::Column, chain: &[&Scope]) -> Resolution { + if let Some(qualifier) = &column.table { + for scope in chain { + if let Some(alias) = source_key(scope, &qualifier.name) { + return self.classify(scope, &alias); + } + } + return Resolution::Unknown; + } + for scope in chain { + let mut resolver = Resolver::new(scope, self.schema, false); + if resolver.is_ambiguous(&column.name.name) { + return Resolution::Unknown; + } + if let Some(alias) = resolver.get_table(&column.name.name) { + return self.classify(scope, &alias); + } + } + Resolution::Unknown + } + + fn classify(&self, scope: &Scope, alias: &str) -> Resolution { + match scope.sources.get(alias) { + Some(source) if self.is_target(&source.expression) => Resolution::Target, + Some(_) => Resolution::Other, + None => Resolution::Unknown, + } + } + + fn is_target(&self, expression: &Expression) -> bool { + match expression { + Expression::Table(table) => { + table.schema.is_none() + && self.target_tables.contains(&table.name.name.to_lowercase()) + } + Expression::Cte(cte) => self.target_ctes.contains(&cte.alias.name.to_lowercase()), + _ => false, + } + } + + fn has_target_source(&self, scope: &Scope) -> bool { + scope + .sources + .values() + .any(|source| self.is_target(&source.expression)) + } + + fn record_stars( + &mut self, + select: &polyglot_sql::expressions::Select, + scope: &Scope, + role: &str, + ) { + for item in &select.expressions { + let covers_target = match star_table(item) { + Some(None) => self.has_target_source(scope), + Some(Some(qualifier)) => source_key(scope, &qualifier) + .and_then(|alias| scope.sources.get(&alias)) + .is_some_and(|source| self.is_target(&source.expression)), + None => false, + }; + if covers_target { + let (start, end) = star_span(item); + self.response.stars.push(StarSite { + start, + end, + scope: role.to_string(), + }); + } + } + } + + fn record_joins( + &mut self, + select: &polyglot_sql::expressions::Select, + scope: &Scope, + role: &str, + ) { + if !self.has_target_source(scope) { + return; + } + for join in &select.joins { + let natural = format!("{:?}", join.kind).starts_with("Natural"); + let using = join + .using + .iter() + .find(|identifier| identifier.name.to_lowercase() == self.column); + if natural || using.is_some() { + let span = using.and_then(|identifier| identifier.span); + self.response.joins.push(SpanSite { + start: span.map(|value| value.start), + end: span.map(|value| value.end), + scope: role.to_string(), + }); + } + } + } + + fn record_outputs(&mut self, select: &polyglot_sql::expressions::Select, role: &str) { + let name = role.strip_prefix(CTE_SCOPE_PREFIX).unwrap_or(role); + if !self.output_ctes.contains(&name.to_lowercase()) { + return; + } + for item in &select.expressions { + match item { + Expression::Alias(alias) if alias.alias.name.to_lowercase() == self.column => { + self.response.outputs.push(OutputSite { + scope: role.to_string(), + start: None, + end: None, + alias_start: alias.alias.span.map(|span| span.start), + alias_end: alias.alias.span.map(|span| span.end), + }); + } + Expression::Column(column) if column.name.name.to_lowercase() == self.column => { + self.response.outputs.push(OutputSite { + scope: role.to_string(), + start: column.span.map(|span| span.start), + end: column.span.map(|span| span.end), + alias_start: None, + alias_end: None, + }); + } + _ if star_table(item).is_some() => { + self.response.star_outputs.push(role.to_string()); + } + _ => {} + } + } + } +} + +enum Resolution { + Target, + Other, + Unknown, +} + +/// Keep only the CTEs a SELECT reads; Polyglot also lists every visible CTE as a source. +fn local_scope(scope: &Scope, select: &polyglot_sql::expressions::Select) -> Scope { + let read: HashSet = select_relation_names(select).into_keys().collect(); + let mut local: Scope = Scope::new(scope.expression.clone()); + local.sources = scope + .sources + .iter() + .filter(|(name, source)| { + !matches!(*source.expression, Expression::Cte(_)) || read.contains(&name.to_lowercase()) + }) + .map(|(name, source)| (name.clone(), source.clone())) + .collect(); + local.lateral_sources = scope.lateral_sources.clone(); + local +} + +fn source_key(scope: &Scope, name: &str) -> Option { + if scope.sources.contains_key(name) { + return Some(name.to_string()); + } + scope + .sources + .keys() + .find(|key| key.eq_ignore_ascii_case(name)) + .cloned() +} + +fn projection_site( + select: &polyglot_sql::expressions::Select, + start: usize, +) -> Option { + for item in &select.expressions { + match item { + Expression::Column(column) if column.span.map(|span| span.start) == Some(start) => { + return Some(ProjectionSite { + alias: None, + alias_start: None, + alias_end: None, + }); + } + Expression::Alias(alias) => { + if let Expression::Column(column) = &alias.this + && column.span.map(|span| span.start) == Some(start) + { + return Some(ProjectionSite { + alias: Some(alias.alias.name.clone()), + alias_start: alias.alias.span.map(|span| span.start), + alias_end: alias.alias.span.map(|span| span.end), + }); + } + } + _ => {} + } + } + None +} diff --git a/crates/sqlbuild-rules-native/src/column_references/main/analyze.rs b/crates/sqlbuild-rules-native/src/column_references/main/analyze.rs new file mode 100644 index 0000000000..d1334fae40 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/main/analyze.rs @@ -0,0 +1,3 @@ +pub(crate) fn analyze_json(request_json: &str) -> Result { + crate::column_references::references::analyze_json(request_json) +} diff --git a/crates/sqlbuild-rules-native/src/column_references/main/mod.rs b/crates/sqlbuild-rules-native/src/column_references/main/mod.rs new file mode 100644 index 0000000000..135b61fdc6 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod analyze; diff --git a/crates/sqlbuild-rules-native/src/column_references/mod.rs b/crates/sqlbuild-rules-native/src/column_references/mod.rs new file mode 100644 index 0000000000..c776b9c603 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod main; +#[path = "_helpers/references.rs"] +mod references; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules-native/src/column_references/tests.rs b/crates/sqlbuild-rules-native/src/column_references/tests.rs new file mode 100644 index 0000000000..0b84c98315 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_column_references.rs"] +mod column_references; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules-native/src/column_references/tests/helpers.rs b/crates/sqlbuild-rules-native/src/column_references/tests/helpers.rs new file mode 100644 index 0000000000..37991a44a9 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/tests/helpers.rs @@ -0,0 +1,58 @@ +use crate::column_references::main::analyze::analyze_json; +use serde_json::{Value, json}; + +pub(crate) fn analyze(sql: &str, output_ctes: &[&str]) -> Result { + let request: Value = json!({ + "sql": sql, + "dialect": "duckdb", + "column": "amount", + "schema": {"strict": false, "tables": [ + {"name": "orders_fact", "columns": [ + {"name": "order_id", "type": "INTEGER"}, + {"name": "amount", "type": "INTEGER"} + ]}, + {"name": "customers", "columns": [ + {"name": "customer_id", "type": "INTEGER"}, + {"name": "order_id", "type": "INTEGER"} + ]} + ]}, + "target_tables": ["orders_fact"], + "output_ctes": output_ctes, + }); + serde_json::from_str(&analyze_json(&request.to_string())?).map_err(|error| error.to_string()) +} + +pub(crate) fn slices(sql: &str, sites: &Value, start: &str, end: &str) -> Vec { + let chars: Vec = sql.chars().collect(); + sites + .as_array() + .map(|items| { + items + .iter() + .filter_map(|item| span_text(&chars, item, start, end)) + .collect() + }) + .unwrap_or_default() +} + +pub(crate) fn strings(sites: &Value, key: &str) -> Vec { + sites + .as_array() + .map(|items| { + items + .iter() + .filter_map(|item| item[key].as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default() +} + +pub(crate) fn count(sites: &Value) -> usize { + sites.as_array().map_or(0, Vec::len) +} + +fn span_text(chars: &[char], item: &Value, start: &str, end: &str) -> Option { + let start: usize = usize::try_from(item[start].as_u64()?).ok()?; + let end: usize = usize::try_from(item[end].as_u64()?).ok()?; + Some(chars[start..end].iter().collect()) +} diff --git a/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs b/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs new file mode 100644 index 0000000000..ec7ab77d68 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs @@ -0,0 +1,226 @@ +use crate::column_references::tests::helpers::{analyze, count, slices, strings}; +use crate::column_references::tests::test_types::ColumnReferenceTestCase; +use serde_json::Value; + +#[test] +fn given_column_uses_when_analyzing_then_reports_rename_sites() -> Result<(), String> { + let test_cases = [ + ColumnReferenceTestCase { + description: "qualified and unqualified uses across clauses", + sql: "SELECT f.amount, SUM(amount) AS total FROM orders_fact f \ + JOIN customers c ON c.order_id = f.order_id WHERE amount > 0 GROUP BY f.amount", + output_ctes: &[], + expected_parsed: true, + expected_names: &["amount", "amount", "amount", "amount"], + expected_scopes: &["root", "root", "root", "root"], + expected_bare_projections: 1, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "same-named column of another relation is not the target", + sql: "SELECT c.amount FROM customers c JOIN orders_fact f ON c.order_id = f.order_id", + output_ctes: &[], + expected_parsed: true, + expected_names: &[], + expected_scopes: &[], + expected_bare_projections: 0, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "pass-through inside a CTE is reported in the CTE scope", + sql: "WITH x AS (SELECT amount FROM orders_fact) SELECT amount FROM x", + output_ctes: &[], + expected_parsed: true, + expected_names: &["amount"], + expected_scopes: &["cte:x"], + expected_bare_projections: 1, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "a star CTE over the target makes its column a target too", + sql: "WITH x AS (SELECT * FROM orders_fact) SELECT x.amount AS a FROM x", + output_ctes: &[], + expected_parsed: true, + expected_names: &["amount"], + expected_scopes: &["root"], + expected_bare_projections: 0, + expected_aliases: &["a"], + expected_star_scopes: &["cte:x"], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "a USING join on the column is reported", + sql: "SELECT order_id FROM orders_fact JOIN customers USING (amount)", + output_ctes: &[], + expected_parsed: true, + expected_names: &[], + expected_scopes: &[], + expected_bare_projections: 0, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 1, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "output scopes report aliased and bare projections of every branch", + sql: "WITH fixture AS (SELECT 1 AS amount UNION ALL SELECT 2 AS amount) \ + SELECT amount FROM orders_fact", + output_ctes: &["fixture", "root"], + expected_parsed: true, + expected_names: &["amount"], + expected_scopes: &["root"], + expected_bare_projections: 1, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &["amount", "amount"], + expected_outputs: 3, + }, + ColumnReferenceTestCase { + description: "a later set-operation branch does not name the output", + sql: "SELECT amount FROM customers UNION ALL SELECT amount FROM orders_fact", + output_ctes: &[], + expected_parsed: true, + expected_names: &["amount"], + expected_scopes: &["root"], + expected_bare_projections: 0, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "the first branch of a nested set operation names the output", + sql: "(SELECT amount FROM orders_fact EXCEPT SELECT amount FROM orders_fact) \ + UNION ALL SELECT * FROM orders_fact", + output_ctes: &[], + expected_parsed: true, + expected_names: &["amount", "amount"], + expected_scopes: &["root", "root"], + expected_bare_projections: 1, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "a star in a later branch does not pass the column through", + sql: "WITH x AS (SELECT 1 AS order_id, 2 AS amount UNION ALL SELECT * FROM orders_fact) \ + SELECT * FROM customers UNION ALL SELECT * FROM orders_fact", + output_ctes: &[], + expected_parsed: true, + expected_names: &[], + expected_scopes: &[], + expected_bare_projections: 0, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "unparseable SQL reports that it was not analysed", + sql: "SELECT FROM WHERE (", + output_ctes: &[], + expected_parsed: false, + expected_names: &[], + expected_scopes: &[], + expected_bare_projections: 0, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ]; + for test_case in &test_cases { + let response: Value = analyze(test_case.sql, test_case.output_ctes)?; + let references: &Value = &response["references"]; + let projections: Vec<&Value> = references + .as_array() + .map(|items| items.iter().map(|item| &item["projection"]).collect()) + .unwrap_or_default(); + let bare: usize = projections + .iter() + .filter(|projection| projection.is_object() && projection["alias"].is_null()) + .count(); + let aliases: Vec = projections + .iter() + .filter_map(|projection| projection["alias"].as_str().map(str::to_string)) + .collect(); + assert_eq!( + response["parsed"], test_case.expected_parsed, + "{}", + test_case.description + ); + assert_eq!( + slices(test_case.sql, references, "nameStart", "nameEnd"), + test_case.expected_names, + "{}", + test_case.description + ); + assert_eq!( + strings(references, "scope"), + test_case.expected_scopes, + "{}", + test_case.description + ); + assert_eq!( + bare, test_case.expected_bare_projections, + "{}", + test_case.description + ); + assert_eq!( + aliases, test_case.expected_aliases, + "{}", + test_case.description + ); + assert_eq!( + strings(&response["stars"], "scope"), + test_case.expected_star_scopes, + "{}", + test_case.description + ); + assert_eq!( + count(&response["joins"]), + test_case.expected_joins, + "{}", + test_case.description + ); + assert_eq!( + slices( + test_case.sql, + &response["outputs"], + "aliasStart", + "aliasEnd" + ), + test_case.expected_output_aliases, + "{}", + test_case.description + ); + assert_eq!( + count(&response["outputs"]), + test_case.expected_outputs, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/column_references/tests/test_types.rs b/crates/sqlbuild-rules-native/src/column_references/tests/test_types.rs new file mode 100644 index 0000000000..0032a0ffae --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/tests/test_types.rs @@ -0,0 +1,14 @@ +pub(crate) struct ColumnReferenceTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) output_ctes: &'static [&'static str], + pub(crate) expected_parsed: bool, + pub(crate) expected_names: &'static [&'static str], + pub(crate) expected_scopes: &'static [&'static str], + pub(crate) expected_bare_projections: usize, + pub(crate) expected_aliases: &'static [&'static str], + pub(crate) expected_star_scopes: &'static [&'static str], + pub(crate) expected_joins: usize, + pub(crate) expected_output_aliases: &'static [&'static str], + pub(crate) expected_outputs: usize, +} diff --git a/crates/sqlbuild-rules-native/src/lib.rs b/crates/sqlbuild-rules-native/src/lib.rs index b37d85783b..056f1dd5f9 100644 --- a/crates/sqlbuild-rules-native/src/lib.rs +++ b/crates/sqlbuild-rules-native/src/lib.rs @@ -1,6 +1,7 @@ #![forbid(unsafe_code)] pub mod bindings; +mod column_references; mod compiler; mod configuration; mod constants; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index 19c6124540..5894fca18c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -1,6 +1,6 @@ --- name: sqlbuild -description: ALWAYS load this skill when doing ANY SQLBuild work - models, sources, seeds, macros, enums, constants, tests, audits, scenarios, incremental and microbatch models, configuration, CLI, adapters, or dbt interop. Covers the verify loop (compile, plan, build), comparing data or two queries with sqb diff, inspecting macro/enum/constant visibility with sqb scope, impact analysis with sqb lineage, and where to find exact syntax. +description: ALWAYS load this skill when doing ANY SQLBuild work - models, sources, seeds, macros, enums, constants, tests, audits, scenarios, incremental and microbatch models, configuration, CLI, adapters, or dbt interop. Covers the verify loop (compile, plan, build), comparing data or two queries with sqb diff, inspecting macro/enum/constant visibility with sqb scope, impact analysis with sqb lineage, renaming or moving models and columns with sqb rename and sqb mv, and where to find exact syntax. --- # SQLBuild @@ -78,6 +78,7 @@ the trigger applies. | Know what a model can see: which macros, enums and constants are visible, used, or unavailable | `sqb scope model:` | | Fix "unknown macro/enum/constant", or decide where a declaration should live | `sqb scope model: --explain macro:` | | Preview whether moving a file breaks declaration visibility | `sqb scope model: --as-path ` | +| Rename a model, move a model file, or rename a column | `sqb rename model: `, `sqb mv model: `, `sqb rename column:. ` | | Find what a model depends on, or what breaks if it changes | `sqb lineage --direction both` | | Trace where a column comes from, or who consumes it before renaming or dropping it | `sqb lineage . --direction downstream` | | See which intermediate models a SQL test will really execute | `sqb test --select --inspect` | @@ -142,6 +143,18 @@ functions, but not in ordinary `MODEL()` config fields. Placement decides visibi **Snapshots** (SCD type 2) use `timestamp` or `check` strategies; full refreshes are policy-gated. +**Renaming and moving.** Never rename or move a model or column by editing files by hand. `sqb +rename` and `sqb mv` rewrite every `__ref`, fixture CTE, header reference and column use from +compiler facts (including YAML and `SCHEMA` relationships), add `migrate_from` whenever a +data-holding relation moves, carry scoped macros, enums, constants and schemas to the folder +placement requires, verify the edited project compiles in a scratch copy, and write nothing if any +step fails. Run with `--dry-run` first. A column rename is one step by default (downstream models +keep their output names via `AS `); `--cascade` renames pass-through outputs downstream too. +When the command refuses, fix the listed locations (usually a macro call, `SELECT *`, or a +declaration file to split) and rerun. Then `sqb plan` to see the migration; on a target that never +built the old name it warns about the missing origin and builds fresh unless the target sets +`missing_migration_origin`. Read [references/docs/cli/rename.md](references/docs/cli/rename.md). + **Rules** are compile-time project checks (`sqb rules list`, `sqb rules show `). `sqb format` rewrites sources deterministically; `sqb format --check` only reports. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 0e700532a3..db76c4d019 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -111,6 +111,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [sqb format](cli/format.md) (`cli/format`) - Apply canonical SQLBuild SQL formatting. - [sqb contract](cli/contract.md) (`cli/contract`) - Compare or generate repository contracts from physical warehouse schemas. - [scope](cli/scope.md) (`cli/scope`) - Inspect declaration visibility, usage, placement, and move impact offline. +- [rename and mv](cli/rename.md) (`cli/rename`) - Rename or move a model, or rename a column, and update every reference in one verified step. - [sqb rules](cli/rules.md) (`cli/rules`) - List, inspect, or run compiler-integrated Rules and generate project guidance. - [sqb plan](cli/plan.md) (`cli/plan`) - Preview what SQLBuild will do before executing. - [sqb build](cli/build.md) (`cli/build`) - Compile, plan, and execute the selected build lifecycle. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 253b81aa58..6d929df51e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -12,6 +12,7 @@ Online: https://sqlbuild.com/docs/cli/janitor/ - Flags - Lifecycle - Compatibility views +- Pending migration origins - Relation age - Audit record - Restoring an archive @@ -74,6 +75,10 @@ A compatibility view is never archived as a stale relation. Once it expires, or Run the same SQLBuild version for `sqb janitor` as for builds. An older janitor does not know compatibility views and archives them as stale relations. +## Pending migration origins + +A relation that a project model still has to migrate from is never archived. While a model declares `migrate_from` and its own relation does not exist yet in the target, the janitor skips the origin and lists it as `pending migration origin for `. A bare `migrate_from` name is matched in each schema the project builds into; a qualified name only at that location. Once the model has built, the origin is handled as usual: after a migration, the old name holds a compatibility view, which expires as described above. + ## Relation age `retention_days` is measured from the last time a relation was written, as reported by the warehouse. For a relation the project no longer builds, that is roughly when it stopped being built. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md new file mode 100644 index 0000000000..132044ab26 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md @@ -0,0 +1,126 @@ + + +# rename and mv + +> Rename or move a model, or rename a column, and update every reference in one verified step. + +Online: https://sqlbuild.com/docs/cli/rename/ + +## Contents + +- Usage +- What changes +- History +- Refusals +- Flags +- Output +- Exit codes +- Limitations + +`sqb rename` and `sqb mv` change a name everywhere the compiler knows it is used, instead of leaving you to find every `__ref`, fixture, and header by hand. Every edit comes from compiled facts, not text search: a column rename follows the columns each query actually binds, so a same-named column of another table is left alone. + +Nothing is written until the edited project compiles. The command applies its edits to a scratch copy of the project, compiles it the way `sqb compile` does, and only then writes the files. If anything cannot be rewritten safely, the command lists those locations and writes nothing. + +## Usage + +```bash +sqb rename model: [flags] +sqb mv model: [flags] +sqb rename column:. [--cascade] [flags] +``` + +```bash +sqb rename model:stg_orders stg_order_lines +sqb mv model:fact_orders models/reporting/ # keeps the name +sqb mv model:fact_orders models/reporting/order_facts.sql # moves and renames +sqb rename column:stg_orders.amount revenue +sqb rename column:stg_orders.amount revenue --cascade +``` + +A model rename keeps the file in its folder and renames it to `.sql`. `sqb mv` takes a new file path, which also sets the model name, or a folder ending in `/`, which keeps it. + +## What changes + +Renaming or moving a model rewrites: + +- every `__ref("")` in models, tests, scenarios, audits, hooks, and SQL functions; +- `__ref__` and `__expected__` fixture CTEs in tests and scenarios; +- `cursor_inputs` keys, `relationships (to ...)` audits, and `__ref("")` inside quoted SQL in `MODEL` headers and reusable `SCHEMA` declarations; +- `relationships` audits (`to:` and `field:`) and `__ref("")` in source and seed YAML. + +Moving a model to another folder also moves the macros, enums, constants, reusable schemas, generic audits, and named hooks it uses when their [placement](../concepts/declaration-scopes/placement.md) must change, following the same rules `sqb compile` enforces: + +- a declaration only the moved model uses moves with it, into the destination folder's matching role; +- a declaration other resources still use moves to the lowest folder they share, for example `models/staging/_sqlbuild/macros/`; +- when that folder would be an authored root such as `models/`, or the users span resource trees such as models and tests, it moves to the top-level role (`macros/`, `enums/`, `constants/`, `schemas/`, ...). + +Declarations are moved, never copied. Folders the move leaves empty are removed. + +Renaming a column rewrites the model that produces it and every query that reads it: + +- the producing projection, as `amount AS revenue`, or the existing alias; +- every downstream reference, including filters, joins, grouping, and ordering; +- column declarations and column-valued config such as `unique_key`, `cursor`, and `partition_by`, and `cursor_inputs` and `relationships (field ...)` entries that point at the column; +- fixture columns of `__ref__` and `__expected__` CTEs. + +By default a column rename is one step: downstream models keep their output names, so `SELECT o.amount` becomes `SELECT o.revenue AS amount`. With `--cascade`, a downstream model that passes the column through unchanged, as a bare column, a same-named alias, or `SELECT *`, renames its output too, and the rename continues through its own consumers. + +## History + +A renamed model keeps its warehouse history through a [model migration](../concepts/models/migrations.md). Whenever the model's relation moves and its materialization keeps warehouse data (table, view, incremental, or snapshot), the command adds `migrate_from ` to the model header, so the move does not depend on automatic discovery finding exactly one match. The next `sqb build` migrates the relation and leaves a compatibility view at the old name. A move that keeps the relation, such as a folder change in the same schema, adds nothing. + +On a target that never built the old name, the origin is missing. By default the plan warns and the model builds fresh; set [`missing_migration_origin`](../concepts/project-configuration.md#missing-migration-origins) on a target to require confirmation or stop the build instead. + +A renamed column of an incremental or snapshot model gets ` (migrate_from )` in its `columns` block, so the next build renames it in place as a [column migration](../concepts/models/column-migrations.md). Table and view models are rebuilt with the new column anyway; they get the declaration only when the model itself declares `migrate_from`. + +The command refuses a rename while the model, or the column, still declares `migrate_from` from an earlier rename. Build it on every target and remove the declaration first. + +## Refusals + +The command refuses, and writes nothing, when: + +- the new name already belongs to another model or column, or the destination file exists; +- a declaration a moved model uses must change folder, but its file also holds declarations that must stay, or it is not an authored project file, or a file with its name already exists at the new folder; +- a move would leave no model in the old schema, so the old relation could not be found; +- a reference is produced by a macro, or a column is read inside SQL that a macro generates; +- a column reaches a model through `SELECT *` in a subquery, a `USING` or `NATURAL` join, or an unqualified name that could belong to several tables; +- a downstream model's `SELECT *` would silently rename its output, without `--cascade`; +- Python code or Python node SQL strings name the model or column. + +Each location is listed with its file, line, and column, with the fix. Edit those locations, or pass the model or column to the macro as an argument, and run the command again. The command never applies part of a rename. + +References produced by a macro are only possible when a project turns off explicit references (`[references] enforce_explicit = false`); such projects get the refusal list for them. + +## Flags + +| Flag | Description | +|------|-------------| +| `--cascade` | For column renames, rename the output of every downstream model that passes the column through. | +| `--dry-run` | Print every edit and verify the edited project compiles, without writing files. | +| `--json` | Print the result as JSON on stdout. Progress goes to stderr. | + +## Output + +```text +Rename model stg_orders -> stg_order_lines +├── models/marts/fact_orders.sql +│ └── 24:13 reference __ref("stg_orders") -> __ref("stg_order_lines") +├── models/staging/stg_order_lines.sql (moved from models/staging/stg_orders.sql) +└── tests/unit/test_fact_orders.sql + └── 4:1 fixture __ref__stg_orders -> __ref__stg_order_lines +Compiled: ok, 3 files changed +``` + +With `--json`, the result has `operation`, `target`, `new_name`, `destination`, `status` (`applied`, `dry_run`, `refused`, or `compile_failed`), `files` with every edit, `migrations`, `renamed_columns`, `manual`, `blocking`, and `compile.errors`. + +## Exit codes + +| Code | Meaning | +|------|---------| +| `0` | Applied, or a dry run that compiles. | +| `1` | Refused, the edited project did not compile, or the request was invalid. No files were changed. | + +## Limitations + +- Descriptions and comments are not rewritten. +- Queries outside the project that read the old relation keep working through the old-name compatibility view until it expires. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md index 85eeb0161a..eef9085a54 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md @@ -21,6 +21,8 @@ Column migrations apply to incremental models, including microbatch models, and ## Declaring a rename +[`sqb rename column:.`](../../cli/rename.md) renames the column in the model and every query that reads it, and adds the `migrate_from` declaration below for you. + Add `migrate_from` to the renamed column in the model header, naming the old column: ```sql @@ -90,7 +92,7 @@ Automatically detected renames are marked `(automatic)`. The decision is based o | `already renamed, record` | The new column exists and the old one doesn't, but no rename is recorded, which is what an interrupted rename leaves behind. Record it without renaming again. | | `done` | The rename is recorded and the new column exists. Nothing to do. | | `conflict` | Both columns exist. The build stops (`M110`); drop one of them, or remove `migrate_from` and let `on_schema_change` handle the columns. | -| `source missing` | Neither column exists and no rename is recorded. The build stops (`M109`); remove `migrate_from` if the rename happened elsewhere or is no longer needed. | +| `source missing` | Neither column exists and no rename is recorded. The plan warns (`M109`) and the columns follow `on_schema_change` as if `migrate_from` were absent. A target can require confirmation or stop the build instead with [`missing_migration_origin`](../project-configuration.md#missing-migration-origins). | | `source still produced` | The query still selects the old column. The build stops (`M111`). | | `unsupported` | The warehouse can't rename this table's columns in place (see [Adapter support](#adapter-support)). The build stops (`M112`). | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md index ed7efd0806..6bce224583 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md @@ -23,7 +23,9 @@ Table and view models can declare a migration too. They hold no history, so they ## Declaring a migration -Add `migrate_from` to the renamed model's header, naming the old model: +[`sqb rename`](../../cli/rename.md) and `sqb mv` rename or move a model, update every reference to it, and add `migrate_from` whenever the model's relation moves and its materialization keeps warehouse data (table, view, incremental, or snapshot). + +To declare a migration by hand, add `migrate_from` to the renamed model's header, naming the old model: ```sql MODEL ( @@ -52,7 +54,7 @@ After the move is recorded, `migrate_from` has no further effect, and SQLBuild t | `superseded replace` | The new relation was itself the origin of the latest recorded move, for example after renaming a model back. Replace it with the newer data. | | `forced replace` | The new relation has its own build history and `migrate_force true` is set. Replace it. | | `conflict` | The new relation has its own build history. The build stops (`M103`) until you set `migrate_force true` or remove `migrate_from`. | -| `origin missing` | The old relation doesn't exist and no move was recorded. The build stops (`M102`). | +| `origin missing` | The old relation doesn't exist and no move was recorded, for example on a target that never built the old name. The plan warns (`M102`) and the model builds fresh. A target can require confirmation or stop the build instead with [`missing_migration_origin`](../project-configuration.md#missing-migration-origins). If SQLBuild has build history for the old relation in this target, it should exist, so the build stops (`M102`) whatever the policy: the connection may lack permission to see it, or it was dropped outside SQLBuild. Restore access, or remove `migrate_from` to build fresh deliberately. | The origin and destination must be compatible under the model's normal `on_schema_change` rules; otherwise the build stops (`M104`). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 0d23f38517..cef8b9084c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -152,6 +152,7 @@ A target is a named build context - the database and schema you build into, plus | `vars` | Target-specific project variables | | `defer_sources_to` | Target name to read managed source data from (see [Loaders](python-nodes/loaders.md#source-deferral)) | | `clone` | Clone policy (see below) | +| `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny` (see below) | ```toml [connections.warehouse] @@ -217,6 +218,25 @@ password = "${ENV:SNOWFLAKE_PASSWORD}" Commands then use `dev` automatically. An explicit command such as `sqb build --target prod` still takes precedence for that invocation. +### Missing migration origins + +A `migrate_from` declaration, on a model or a column, names a relation or column that exists in the targets where the rename is pending. A target that never built the old name, such as a fresh development schema, has nothing to migrate. `missing_migration_origin` decides what happens there: + +| Value | Behavior | +|-------|----------| +| `allow` | Default. The plan warns (`M102` for models, `M109` for columns), nothing is migrated, and the model builds as if `migrate_from` were absent. | +| `require_confirmation` | The build asks for confirmation on a terminal, and stops otherwise; pass `--allow-missing-migration-origin` to confirm in non-interactive runs. | +| `deny` | The build stops with `M102` or `M109`. Use it for targets where a missing origin means the wrong target or a lost relation. | + +```toml +[targets.prod] +missing_migration_origin = "deny" +``` + +`sqlbuild_local.toml` can override the value for a target. + +The policy covers only origins with no build history in the target. A model origin that SQLBuild built in this target but can no longer see, because of permissions or because it was dropped outside SQLBuild, always stops the build with `M102`. Column origins have no such history, so `M109` always follows the policy. `sqb dbt run` and `sqb dbt build` apply the policy too, but have no confirmation flag: they ask on a terminal and stop otherwise. In non-interactive dbt runs, build the affected models with `sqb build --allow-missing-migration-origin` and leave them out of the dbt selection, or set the target's `missing_migration_origin` to `allow`. + ### Clone policies Targets can declare whether they allow cloning to or from: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f4a8b177bd..37bd4c9921 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -8,6 +8,7 @@ class LintPreparationRequest(TypedDict): prior_sites: list[int] dialect: str +def analyze_column_references_json(request_json: str) -> str: ... def catalogue_json() -> str: ... def evaluate_json(request_json: str) -> str: ... def finalize_rule_findings_json(request_json: str) -> str: ... diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py b/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py new file mode 100644 index 0000000000..7080ea11cb --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py @@ -0,0 +1,32 @@ +"""Typed confirmation shared by the build safety policies.""" + +from __future__ import annotations + +from typing import TextIO + +from sqlbuild.cli.commands.exceptions import CliUserError + + +def confirm_typed_action( + *, + action: str, + non_interactive_help: str, + warning: str, + expected: str, + input_stream: TextIO, + output_stream: TextIO, + code: str | None = None, +) -> None: + """Ask the user to type the expected text, or fail when the run is not interactive.""" + + if not input_stream.isatty(): + raise CliUserError( + f"{action} requires confirmation", + code=code, + help=non_interactive_help, + ) + _ = output_stream.write(f"{warning}\n\n") + _ = output_stream.write(f"Type `{expected}` to continue: ") + output_stream.flush() + if input_stream.readline().strip() != expected: + raise CliUserError(f"{action} cancelled", code=code) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py index 5c8db4d7ee..152ea7bb45 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py @@ -3,7 +3,9 @@ from __future__ import annotations import sys +from typing import TextIO +from sqlbuild.cli.commands._helpers.build_planning.confirmation import confirm_typed_action from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( enforce_model_execution_limit, executable_model_count, @@ -17,9 +19,11 @@ from sqlbuild.cli.commands._helpers.build_planning.table_type import ( enforce_table_type_downgrade_policy, ) +from sqlbuild.cli.commands.constants import MISSING_ORIGIN_BUILD_HELP from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import BuildCommandRequest, BuildInvocation from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision +from sqlbuild.compiler.planner.constants import HIDDEN_ORIGIN_REMEDY from sqlbuild.compiler.planner.models import ( ColumnMigrationPlanEntry, ModelMigrationPlanEntry, @@ -27,6 +31,7 @@ PlanWarning, ) from sqlbuild.compiler.planner.types import WarningSeverity +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy _OLD_NAME_CODES: frozenset[str] = frozenset({"M114", "P008"}) _COLUMN_BLOCK_CODES: dict[ColumnMigrationDecision, str] = { @@ -42,8 +47,13 @@ def enforce_build_plan_policies( ) -> None: """Apply migration, execution-limit, and storage safety gates in their required order.""" - _enforce_model_migration_policy(plan=plan) - _enforce_column_migration_policy(plan=plan) + enforce_migration_plan_policies( + plan=plan, + allow_missing_migration_origin=request.allow_missing_migration_origin, + non_interactive_help=MISSING_ORIGIN_BUILD_HELP, + input_stream=sys.stdin, + output_stream=sys.stdout, + ) _enforce_old_name_policy(plan=plan) enforce_model_execution_limit( model_count=executable_model_count(plan=plan), @@ -71,6 +81,27 @@ def enforce_build_plan_policies( ) +def enforce_migration_plan_policies( + *, + plan: PlanOutput, + allow_missing_migration_origin: bool, + non_interactive_help: str, + input_stream: TextIO, + output_stream: TextIO, +) -> None: + """Refuse blocked model and column migrations, then confirm building past missing origins.""" + + _enforce_model_migration_policy(plan=plan) + _enforce_column_migration_policy(plan=plan) + _confirm_missing_origins( + plan=plan, + allow_missing_migration_origin=allow_missing_migration_origin, + non_interactive_help=non_interactive_help, + input_stream=input_stream, + output_stream=output_stream, + ) + + def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: """Refuse to build while any planned migration conflicts or is incompatible.""" @@ -79,19 +110,27 @@ def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: ) if not blocked: return + hidden: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in blocked if entry.origin_hidden + ) + if hidden: + raise CliUserError( + "model migration origin was built in this target but is not visible for " + + _origins(hidden) + + f"; {HIDDEN_ORIGIN_REMEDY}", + code="M102", + help="Run sqb plan to see every model migration decision.", + ) missing: tuple[ModelMigrationPlanEntry, ...] = tuple( entry for entry in blocked if entry.decision == MigrationDecision.ORIGIN_MISSING ) if missing: raise CliUserError( "model migration origin does not exist for " - + ", ".join( - f"'{entry.model_name}' (migrate_from " - f"{entry.origin.qualified_name or entry.origin.name})" - for entry in missing - ) - + " and no recorded migration into it was found; if the migration already happened " - "elsewhere or is no longer needed, remove migrate_from from the model header", + + _origins(missing) + + " and no recorded migration into it was found, and this target sets " + "missing_migration_origin = deny; if the migration already happened elsewhere or is " + "no longer needed, remove migrate_from from the model header", code="M102", help="Run sqb plan to see every model migration decision.", ) @@ -116,6 +155,13 @@ def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: ) +def _origins(entries: tuple[ModelMigrationPlanEntry, ...]) -> str: + return ", ".join( + f"'{entry.model_name}' (migrate_from {entry.origin.qualified_name or entry.origin.name})" + for entry in entries + ) + + def _enforce_column_migration_policy(*, plan: PlanOutput) -> None: """Refuse to build while any declared column rename cannot be applied safely.""" @@ -136,6 +182,46 @@ def _enforce_column_migration_policy(*, plan: PlanOutput) -> None: ) +def _confirm_missing_origins( + *, + plan: PlanOutput, + allow_missing_migration_origin: bool, + non_interactive_help: str, + input_stream: TextIO, + output_stream: TextIO, +) -> None: + """Confirm before building past declared migrations whose origin is missing.""" + + names: tuple[str, ...] = tuple( + sorted( + { + entry.model_name + for entry in (*plan.migration_entries, *plan.column_migration_entries) + if entry.origin_missing + and not entry.blocks_build + and entry.missing_origin_policy == MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION + } + ) + ) + if not names or allow_missing_migration_origin: + return + listed: str = ", ".join(f"'{name}'" for name in names) + confirm_typed_action( + action=f"building {listed} without its missing migrate_from origin", + non_interactive_help=non_interactive_help, + warning=f"The migrate_from origin of {listed} does not exist in this target; nothing " + "will be migrated.", + expected=( + f"build {names[0]} without its migration" + if len(names) == 1 + else f"build {len(names)} models without their migrations" + ), + input_stream=input_stream, + output_stream=output_stream, + code="M102", + ) + + def _enforce_old_name_policy(*, plan: PlanOutput) -> None: """Refuse to build over a compatibility view or while code reads an old name.""" diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py index f38709f558..3e8c27f173 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py @@ -7,6 +7,9 @@ from typing import TextIO from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.cli.commands._helpers.build_planning.plan_policies import ( + enforce_migration_plan_policies, +) from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( enforce_retention_decrease_policy, ) @@ -14,6 +17,7 @@ BuildProgressCallbacks, format_build_footer, ) +from sqlbuild.cli.commands.constants import MISSING_ORIGIN_DBT_HELP from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter from sqlbuild.cli.progress.main._execution_header import format_execution_header @@ -50,6 +54,13 @@ def execute_sqlbuild_build_work( adapter_name: str = context.adapter_name output_stream: TextIO = context.output_stream use_color: bool = context.use_color + enforce_migration_plan_policies( + plan=plan_output, + allow_missing_migration_origin=False, + non_interactive_help=MISSING_ORIGIN_DBT_HELP, + input_stream=sys.stdin, + output_stream=output_stream, + ) enforce_retention_decrease_policy( plan=plan_output, allow_retention_decrease=False, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index 200ed070e9..895d4d6c6d 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -164,6 +164,7 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) allow_snapshot_full_refresh=args.allow_snapshot_full_refresh, allow_table_type_downgrade=args.allow_table_type_downgrade, allow_retention_decrease=args.allow_retention_decrease, + allow_missing_migration_origin=args.allow_missing_migration_origin, allow_snapshot_schema_change=args.allow_snapshot_schema_change, concurrency=args.concurrency, select=select, @@ -422,6 +423,24 @@ def _dispatch_compile_or_contract( project_dir: Path | None, select: tuple[str, ...], ) -> int | None: + if args.command in {CliCommand.RENAME, CliCommand.MV}: + from sqlbuild.cli.commands.models import RefactorCommandRequest + + if handlers.run_refactor is None or args.refactor_target is None: + raise CliUserError(f"{args.command} command handler is unavailable", code="C950") + return handlers.run_refactor( + request=RefactorCommandRequest( + command=str(args.command), + target=args.refactor_target, + new_name=args.refactor_new_name, + destination=args.refactor_destination, + project_dir=project_dir, + cascade=args.refactor_cascade, + dry_run=args.dry_run, + json_output=args.json, + no_color=args.no_color, + ) + ) if args.command == CliCommand.CONTRACT: from sqlbuild.cli.commands.models import ContractCommandRequest diff --git a/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py b/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py index 261ede787a..386aa017f6 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py @@ -89,6 +89,10 @@ def build_lazy_cli_handlers() -> CliEntrypointHandlers: module_name="sqlbuild.cli.commands.main.inspection._scope", function_name="run_scope", ), + "refactor": _lazy_handler( + module_name="sqlbuild.cli.commands.main.project._refactor", + function_name="run_refactor", + ), "contract": _lazy_handler( module_name="sqlbuild.cli.commands.main.inspection._contract", function_name="run_contract", @@ -220,6 +224,7 @@ def build_lazy_cli_handlers() -> CliEntrypointHandlers: run_rules=lazy["rules"], run_scope=lazy["scope"], run_contract=lazy["contract"], + run_refactor=lazy["refactor"], ) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index 23a39ed79d..d6c353f58c 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -68,6 +68,7 @@ def build_cli_parser(*, use_color: bool = False) -> argparse.ArgumentParser: _add_data_parsers(subparsers) _add_inspection_parsers(subparsers) _add_contract_parser(subparsers) + _add_refactor_parsers(subparsers) _add_maintenance_parsers(subparsers) _add_workspace_parsers(subparsers) _add_dbt_parsers(subparsers) @@ -106,6 +107,47 @@ def _add_contract_parser( action_parser.add_argument("--overwrite", action="store_true") +def _add_refactor_parsers( + subparsers: argparse._SubParsersAction[argparse.ArgumentParser], +) -> None: + rename_parser: argparse.ArgumentParser = subparsers.add_parser( + CliCommand.RENAME, help="rename a model or column and update every reference" + ) + rename_parser.add_argument( + "refactor_target", metavar="TARGET", help="model: or column:." + ) + rename_parser.add_argument("refactor_new_name", metavar="NEW_NAME") + rename_parser.add_argument( + "--cascade", + dest="refactor_cascade", + action="store_true", + default=False, + help="rename a column through every downstream model that passes it through", + ) + move_parser: argparse.ArgumentParser = subparsers.add_parser( + CliCommand.MV, help="move a model file, optionally renaming it, and update references" + ) + move_parser.add_argument("refactor_target", metavar="TARGET", help="model:") + move_parser.add_argument( + "refactor_destination", metavar="DESTINATION", help="new .sql path or folder/" + ) + parser: argparse.ArgumentParser + for parser in (rename_parser, move_parser): + parser.add_argument( + "--dry-run", + dest="dry_run", + action="store_true", + default=False, + help="print every edit and verify it compiles, without writing files", + ) + parser.add_argument( + "--json", + action="store_true", + default=False, + help="print the result as JSON on stdout; progress goes to stderr", + ) + + def _add_compile_and_dag_parsers( subparsers: argparse._SubParsersAction[argparse.ArgumentParser], ) -> None: diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser_arguments.py b/src/sqlbuild/cli/commands/_helpers/entry/parser_arguments.py index 73036ec8e2..bbf8b91063 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser_arguments.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser_arguments.py @@ -63,6 +63,7 @@ def add_execution_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--allow-snapshot-full-refresh", action="store_true", default=False) parser.add_argument("--allow-table-type-downgrade", action="store_true", default=False) parser.add_argument("--allow-retention-decrease", action="store_true", default=False) + parser.add_argument("--allow-missing-migration-origin", action="store_true", default=False) parser.add_argument("--allow-snapshot-schema-change", action="store_true", default=False) parser.add_argument("--concurrency", type=int, default=None) parser.add_argument("--verbose", "-v", action="store_true", default=False) diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py b/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py new file mode 100644 index 0000000000..d3461aba0d --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py @@ -0,0 +1 @@ +"""Helpers for the rename and mv commands.""" diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/command.py b/src/sqlbuild/cli/commands/_helpers/refactor/command.py new file mode 100644 index 0000000000..bb37351b9a --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/refactor/command.py @@ -0,0 +1,230 @@ +"""Run `sqb rename` and `sqb mv`: plan from compiler facts, verify, then write.""" + +from __future__ import annotations + +import sys +import tempfile +import time +from dataclasses import replace +from pathlib import Path +from typing import TextIO + +from sqlbuild.cli.commands._helpers.refactor.compile_facts import compile_for_refactor +from sqlbuild.cli.commands._helpers.refactor.output import ( + render_refactor_json, + render_refactor_text, +) +from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.cli.commands.models import RefactorCommandRequest, RefactorCompile +from sqlbuild.cli.commands.types import CliCommand +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.refactoring.constants import ( + COLUMN_KIND_PREFIX, + COLUMN_TARGET_SEPARATOR, + MODEL_KIND_PREFIX, +) +from sqlbuild.compiler.refactoring.exceptions import RefactorInputError +from sqlbuild.compiler.refactoring.main.commit_refactor_plan import commit_refactor_plan +from sqlbuild.compiler.refactoring.main.plan_column_rename import plan_column_rename +from sqlbuild.compiler.refactoring.main.plan_model_refactor import plan_model_refactor +from sqlbuild.compiler.refactoring.main.stage_refactor_plan import stage_refactor_plan +from sqlbuild.compiler.refactoring.main.with_model_migration import with_model_migration +from sqlbuild.compiler.refactoring.models import RefactorPlan, RefactorRequest +from sqlbuild.compiler.refactoring.types import RefactorOperation, RefactorStatus +from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter +from sqlbuild.presentation.main.supports_color import supports_color + +_STAGING_PREFIX: str = "sqb-refactor-" + + +def run_refactor_command(*, request: RefactorCommandRequest) -> int: + """Plan, verify in a scratch copy, and write one rename or move.""" + + project_dir: Path = (request.project_dir or Path.cwd()).resolve() + stream: TextIO = sys.stderr if request.json_output else sys.stdout + use_color: bool = not request.no_color and supports_color() + status: TransientStatusReporter = TransientStatusReporter(stream=stream, use_color=use_color) + try: + return _run(request=request, project_dir=project_dir, status=status, use_color=use_color) + except RefactorInputError as error: + status.close() + raise CliUserError(error.message, code=error.code, help=error.help) from error + finally: + status.close() + + +def _run( + *, + request: RefactorCommandRequest, + project_dir: Path, + status: TransientStatusReporter, + use_color: bool, +) -> int: + started: float = time.monotonic() + status.start("Compiling project...") + before: RefactorCompile = compile_for_refactor(project_dir=project_dir) + if before.project is None or before.errors: + status.error("Project does not compile; fix it before renaming (sqb compile)") + _write_errors(errors=before.errors) + return 1 + status.complete(message=f"Compiled project. ({time.monotonic() - started:.2f}s)") + refactor_request: RefactorRequest = _refactor_request(request=request) + status.start("Planning edits...") + plan: RefactorPlan = ( + plan_column_rename(project=before.project, request=refactor_request) + if refactor_request.operation == RefactorOperation.RENAME_COLUMN + else plan_model_refactor(project=before.project, request=refactor_request) + ) + status.complete(message=f"Planned edits to {len(plan.changes)} files.") + if plan.blocking or plan.manual: + status.error("Refused: some references cannot be rewritten safely.") + return _finish( + request=request, + plan=plan, + status=RefactorStatus.REFUSED, + diagnostics=(), + use_color=use_color, + ) + with tempfile.TemporaryDirectory(prefix=_STAGING_PREFIX) as staging: + staging_dir: Path = Path(staging) + status.start("Verifying the edited project compiles...") + originals: dict[str, str] = stage_refactor_plan( + project_dir=project_dir, staging_dir=staging_dir, plan=plan + ) + after: RefactorCompile = compile_for_refactor(project_dir=staging_dir, no_cache=True) + if refactor_request.operation != RefactorOperation.RENAME_COLUMN: + migrated: RefactorPlan = with_model_migration( + plan=plan, + before=before.project.graph.project, + after=after.project.graph.project if after.project is not None else None, + originals=originals, + ) + if migrated.blocking: + status.error("Refused: the renamed model's history cannot be kept.") + return _finish( + request=request, + plan=migrated, + status=RefactorStatus.REFUSED, + diagnostics=(), + use_color=use_color, + ) + if migrated.migrations != plan.migrations: + plan = migrated + originals = stage_refactor_plan( + project_dir=project_dir, staging_dir=staging_dir, plan=plan, copy_inputs=False + ) + after = compile_for_refactor(project_dir=staging_dir, no_cache=True) + diagnostics: tuple[CompilerDiagnostic, ...] = _relative( + errors=after.errors, staging_dir=staging_dir + ) + if diagnostics: + status.error("Verification failed: the edited project does not compile.") + return _finish( + request=request, + plan=plan, + status=RefactorStatus.COMPILE_FAILED, + diagnostics=diagnostics, + use_color=use_color, + ) + status.complete(message="Verified: the edited project compiles.") + if request.dry_run: + return _finish( + request=request, + plan=plan, + status=RefactorStatus.DRY_RUN, + diagnostics=(), + use_color=use_color, + ) + status.start(f"Writing {len(plan.changes)} files...") + commit_refactor_plan(project_dir=project_dir, originals=originals, plan=plan) + status.complete(message=f"Wrote {len(plan.changes)} files.") + return _finish( + request=request, + plan=plan, + status=RefactorStatus.APPLIED, + diagnostics=(), + use_color=use_color, + ) + + +def _finish( + *, + request: RefactorCommandRequest, + plan: RefactorPlan, + status: RefactorStatus, + diagnostics: tuple[CompilerDiagnostic, ...], + use_color: bool, +) -> int: + output: str = ( + render_refactor_json(plan=plan, status=status, diagnostics=diagnostics) + if request.json_output + else render_refactor_text( + plan=plan, status=status, diagnostics=diagnostics, use_color=use_color + ) + ) + _ = sys.stdout.write(output) + return 0 if status in {RefactorStatus.APPLIED, RefactorStatus.DRY_RUN} else 1 + + +def _refactor_request(*, request: RefactorCommandRequest) -> RefactorRequest: + target: str = request.target.strip() + if request.command == CliCommand.MV: + if not target.startswith(MODEL_KIND_PREFIX): + raise CliUserError( + f"sqb mv moves models; got '{target}'", + code="C954", + help="write the target as model:", + ) + return RefactorRequest( + operation=RefactorOperation.MOVE_MODEL, + model_name=target.removeprefix(MODEL_KIND_PREFIX), + new_name="", + destination=request.destination or "", + ) + new_name: str = (request.new_name or "").strip() + if target.startswith(MODEL_KIND_PREFIX): + if request.cascade: + raise CliUserError("--cascade applies to column renames only", code="C954") + return RefactorRequest( + operation=RefactorOperation.RENAME_MODEL, + model_name=target.removeprefix(MODEL_KIND_PREFIX), + new_name=new_name, + ) + if target.startswith(COLUMN_KIND_PREFIX) and COLUMN_TARGET_SEPARATOR in target: + model_name: str + column_name: str + model_name, column_name = target.removeprefix(COLUMN_KIND_PREFIX).rsplit( + COLUMN_TARGET_SEPARATOR, 1 + ) + return RefactorRequest( + operation=RefactorOperation.RENAME_COLUMN, + model_name=model_name, + column_name=column_name, + new_name=new_name, + cascade=request.cascade, + ) + raise CliUserError( + f"cannot rename '{target}'", + code="C954", + help="write the target as model: or column:.", + ) + + +def _relative( + *, errors: tuple[CompilerDiagnostic, ...], staging_dir: Path +) -> tuple[CompilerDiagnostic, ...]: + return tuple( + replace(error, path=error.path.relative_to(staging_dir)) + if error.path is not None + and error.path.is_absolute() + and error.path.is_relative_to(staging_dir) + else error + for error in errors + ) + + +def _write_errors(*, errors: tuple[CompilerDiagnostic, ...]) -> None: + error: CompilerDiagnostic + for error in errors: + location: str = f"{error.path.as_posix()}: " if error.path is not None else "" + print(f"error[{error.code}] {location}{error.message}", file=sys.stderr) diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py new file mode 100644 index 0000000000..5bc0c1e0a1 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py @@ -0,0 +1,79 @@ +"""Offline compiles that feed and verify a refactoring.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.cli.commands._helpers.compile.pipeline import analyze_compile_project +from sqlbuild.cli.commands._helpers.compile.target_writer import ( + static_sql_test_planning_diagnostics, +) +from sqlbuild.cli.commands.models import RefactorCompile +from sqlbuild.cli.commands.types import CompileLineageMode +from sqlbuild.cli.compile.models import CompileAnalysis, CompileProfileFlags +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompileAnalysisSelection, CompilerDiagnostic +from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity +from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.compiler.refactoring.models import RefactorProject +from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild.spec.contracts.exceptions import SpecConfigError + + +def compile_for_refactor(*, project_dir: Path, no_cache: bool = False) -> RefactorCompile: + """Compile a project the way `sqb compile` does and collect every error.""" + + try: + analysis: CompileAnalysis = analyze_compile_project( + project_dir=project_dir, + no_sql_validation=False, + selected_target=None, + lineage_mode=CompileLineageMode.NONE, + cli_vars=None, + profile_flags=CompileProfileFlags(), + analysis_selection=CompileAnalysisSelection(no_cache=no_cache), + status=None, + ) + except (ValueError, RulesError, SpecConfigError) as error: + return RefactorCompile( + project=None, + errors=( + CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=str(getattr(error, "code", "E001")), + message=str(getattr(error, "message", str(error))), + ), + ), + ) + errors: list[CompilerDiagnostic] = [ + diagnostic for diagnostic in analysis.diagnostics if diagnostic.is_error + ] + if not errors: + try: + errors.extend( + diagnostic + for diagnostic in static_sql_test_planning_diagnostics( + target_dir=project_dir / "target", + adapter=analysis.adapter, + project=analysis.graph.project, + ) + if diagnostic.is_error + ) + except (CompileInputError, PlannerInputError) as error: + errors.append( + CompilerDiagnostic( + phase=DiagnosticPhase.TEST, + severity=DiagnosticSeverity.ERROR, + code=error.code, + message=str(error), + ) + ) + return RefactorCompile( + project=RefactorProject( + project_dir=project_dir, + graph=analysis.graph, + discovered=analysis.discovered_inputs, + ), + errors=tuple(errors), + ) diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/output.py b/src/sqlbuild/cli/commands/_helpers/refactor/output.py new file mode 100644 index 0000000000..7c3a5317c7 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/refactor/output.py @@ -0,0 +1,228 @@ +"""Human and JSON output for `sqb rename` and `sqb mv`.""" + +from __future__ import annotations + +import json + +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.refactoring.constants import OPERATION_TITLES +from sqlbuild.compiler.refactoring.models import ( + FileChange, + ManualLocation, + MigrationDeclaration, + RefactorPlan, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import RefactorOperation, RefactorStatus +from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild.presentation.main.tree_connector import tree_connector + +_CONTINUE: str = "\u2502 " +_BLANK: str = " " +_POSITION_WIDTH: int = 8 +_KIND_WIDTH: int = 10 +_EXCERPT_LIMIT: int = 60 + + +def render_refactor_text( + *, + plan: RefactorPlan, + status: RefactorStatus, + diagnostics: tuple[CompilerDiagnostic, ...], + use_color: bool, +) -> str: + """Render the edit tree, manual locations, and one terminal status line.""" + + style: CliStyle = CliStyle(use_color=use_color) + lines: list[str] = [style.title(_heading(plan))] + blocked: bool = status == RefactorStatus.REFUSED + if not blocked or not (plan.blocking or plan.manual): + lines.extend(_change_lines(style=style, changes=plan.changes)) + if plan.migrations and not blocked: + lines.append(style.section("Migrations")) + lines.extend(_migration_lines(style=style, migrations=plan.migrations)) + if blocked: + lines.append(style.error_strong(f"Cannot {_verb(plan)} {_target(plan)} automatically")) + lines.extend(_location_lines(style=style, locations=(*plan.blocking, *plan.manual))) + if plan.help and plan.manual: + lines.append(plan.help) + if diagnostics: + lines.append(style.error_strong("Compile errors in the edited project")) + lines.extend(_diagnostic_lines(style=style, diagnostics=diagnostics)) + lines.append(_status_line(style=style, plan=plan, status=status)) + return "\n".join(lines) + "\n" + + +def render_refactor_json( + *, + plan: RefactorPlan, + status: RefactorStatus, + diagnostics: tuple[CompilerDiagnostic, ...], +) -> str: + """Render the same facts as stable JSON.""" + + payload: dict[str, object] = { + "operation": plan.request.operation.value, + "target": _target(plan), + "new_name": plan.request.new_name, + "destination": plan.request.destination, + "cascade": plan.request.cascade, + "status": status.value, + "files_changed": len(plan.changes) if status == RefactorStatus.APPLIED else 0, + "files": [_file_payload(change) for change in plan.changes], + "migrations": [ + { + "model": migration.model_name, + "declaration": migration.declaration, + "reason": migration.reason, + } + for migration in plan.migrations + ], + "renamed_columns": [ + {"model": model, "from": old, "to": new} for model, old, new in plan.renamed_columns + ], + "manual": [_location_payload(item) for item in plan.manual], + "blocking": [_location_payload(item) for item in plan.blocking], + "compile": { + "ok": not diagnostics, + "errors": [ + { + "code": diagnostic.code, + "message": diagnostic.message, + "path": diagnostic.path.as_posix() if diagnostic.path is not None else None, + "line": diagnostic.line, + } + for diagnostic in diagnostics + ], + }, + } + return json.dumps(payload, indent=2) + "\n" + + +def _file_payload(change: FileChange) -> dict[str, object]: + return { + "path": change.path, + "from_path": change.original_path if change.moved else None, + "edits": [ + _edit_payload(edit) for edit in sorted(change.edits, key=lambda item: item.start) + ], + } + + +def _edit_payload(edit: TextEdit) -> dict[str, object]: + return { + "line": edit.line, + "column": edit.column, + "kind": edit.kind.value, + "before": edit.before, + "after": edit.after, + } + + +def _heading(plan: RefactorPlan) -> str: + title: str = OPERATION_TITLES[plan.request.operation] + if plan.request.operation == RefactorOperation.RENAME_COLUMN: + return ( + f"{title} {plan.request.model_name}.{plan.request.column_name} -> " + f"{plan.request.new_name}" + ) + if plan.request.operation == RefactorOperation.MOVE_MODEL: + return f"{title} {plan.request.model_name} -> {plan.request.destination}" + return f"{title} {plan.request.model_name} -> {plan.request.new_name}" + + +def _verb(plan: RefactorPlan) -> str: + return "move" if plan.request.operation == RefactorOperation.MOVE_MODEL else "rename" + + +def _target(plan: RefactorPlan) -> str: + if plan.request.operation == RefactorOperation.RENAME_COLUMN: + return f"column:{plan.request.model_name}.{plan.request.column_name}" + return f"model:{plan.request.model_name}" + + +def _change_lines(*, style: CliStyle, changes: tuple[FileChange, ...]) -> list[str]: + lines: list[str] = [] + index: int + change: FileChange + for index, change in enumerate(changes): + last: bool = index == len(changes) - 1 + moved: str = style.muted(f" (moved from {change.original_path})") if change.moved else "" + lines.append(f"{tree_connector(style=style, last=last)} {change.path}{moved}") + prefix: str = _BLANK if last else _CONTINUE + edits: list[TextEdit] = sorted(change.edits, key=lambda item: item.start) + edit_index: int + edit: TextEdit + for edit_index, edit in enumerate(edits): + connector: str = tree_connector(style=style, last=edit_index == len(edits) - 1) + position: str = f"{edit.line}:{edit.column}".ljust(_POSITION_WIDTH) + lines.append( + f"{style.muted(prefix)}{connector} {position}{edit.kind.value.ljust(_KIND_WIDTH)}" + f"{_excerpt(edit.before)} -> {_excerpt(edit.after)}" + if edit.before + else f"{style.muted(prefix)}{connector} {position}" + f"{edit.kind.value.ljust(_KIND_WIDTH)}+ {_excerpt(edit.after)}" + ) + return lines + + +def _migration_lines(*, style: CliStyle, migrations: tuple[MigrationDeclaration, ...]) -> list[str]: + return [ + f"{tree_connector(style=style, last=index == len(migrations) - 1)} " + f"{style.object_name(item.model_name)} {item.declaration} {style.muted(item.reason)}" + for index, item in enumerate(migrations) + ] + + +def _location_lines(*, style: CliStyle, locations: tuple[ManualLocation, ...]) -> list[str]: + return [ + f"{tree_connector(style=style, last=index == len(locations) - 1)} " + f"{_position(item)} {item.reason}" + for index, item in enumerate(locations) + ] + + +def _diagnostic_lines(*, style: CliStyle, diagnostics: tuple[CompilerDiagnostic, ...]) -> list[str]: + return [ + f"{tree_connector(style=style, last=index == len(diagnostics) - 1)} " + f"{diagnostic.code} " + + ( + f"{diagnostic.path.as_posix()}:{diagnostic.line or ''} " + if diagnostic.path is not None + else "" + ) + + diagnostic.message.splitlines()[0] + for index, diagnostic in enumerate(diagnostics) + ] + + +def _status_line(*, style: CliStyle, plan: RefactorPlan, status: RefactorStatus) -> str: + count: int = len(plan.changes) + files: str = f"{count} file{'s' if count != 1 else ''}" + if status == RefactorStatus.APPLIED: + return style.success_strong(f"Compiled: ok, {files} changed") + if status == RefactorStatus.DRY_RUN: + return style.success_strong(f"Dry run: compiles, {files} would change; nothing written") + if status == RefactorStatus.COMPILE_FAILED: + return style.error_strong("Compiled: failed; no files changed") + return style.error_strong("Refused; no files changed") + + +def _position(location: ManualLocation) -> str: + if location.line is None: + return location.path + return f"{location.path}:{location.line}:{location.column or 1}" + + +def _location_payload(location: ManualLocation) -> dict[str, object]: + return { + "path": location.path, + "line": location.line, + "column": location.column, + "reason": location.reason, + } + + +def _excerpt(text: str) -> str: + single: str = " ".join(text.split()) + return single if len(single) <= _EXCERPT_LIMIT else single[: _EXCERPT_LIMIT - 1] + "\u2026" diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index e4cac9dbda..13d2eb55bf 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -56,6 +56,7 @@ "allow_snapshot_full_refresh": False, "allow_table_type_downgrade": False, "allow_retention_decrease": False, + "allow_missing_migration_origin": False, "allow_snapshot_schema_change": False, "concurrency": None, "verbose": False, @@ -160,6 +161,10 @@ "scope_page_size": 100, "scope_after": None, "scope_paths": "relative", + "refactor_target": None, + "refactor_new_name": None, + "refactor_destination": None, + "refactor_cascade": False, } @@ -216,6 +221,7 @@ class CliNamespace: allow_snapshot_full_refresh: bool allow_table_type_downgrade: bool allow_retention_decrease: bool + allow_missing_migration_origin: bool allow_snapshot_schema_change: bool concurrency: int | None verbose: bool @@ -238,6 +244,10 @@ class CliNamespace: query_limit: int | None query_no_limit: bool cost_selector: str + refactor_target: str | None + refactor_new_name: str | None + refactor_destination: str | None + refactor_cascade: bool cost_limit: int | None cost_no_limit: bool cost_sort: str | None diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index c2d025fcca..ad8a34121e 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -103,4 +103,12 @@ QUERY_DIFF_INCOMPLETE_PREPARATION_CODES: frozenset[str] = frozenset( {"C229", "C230", "C231", "C232", "C233", "C234", "C235", "C240", "C244"} ) +MISSING_ORIGIN_BUILD_HELP: str = ( + "Pass --allow-missing-migration-origin to confirm in non-interactive runs." +) +MISSING_ORIGIN_DBT_HELP: str = ( + "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected " + "models with sqb build --allow-missing-migration-origin and leave them out of this dbt " + "selection, or set the target's missing_migration_origin to 'allow'." +) QUERY_DIFF_INCOMPLETE_EXECUTION_CODES: frozenset[str] = frozenset({"C237", "C238", "C239"}) diff --git a/src/sqlbuild/cli/commands/main/project/_refactor.py b/src/sqlbuild/cli/commands/main/project/_refactor.py new file mode 100644 index 0000000000..9752d45334 --- /dev/null +++ b/src/sqlbuild/cli/commands/main/project/_refactor.py @@ -0,0 +1,10 @@ +"""CLI entry for `sqb rename` and `sqb mv`.""" + +from sqlbuild.cli.commands._helpers.refactor.command import run_refactor_command +from sqlbuild.cli.commands.models import RefactorCommandRequest + + +def run_refactor(*, request: RefactorCommandRequest) -> int: + """Run one rename or move.""" + + return run_refactor_command(request=request) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 74abf7baf5..be2c086de4 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -52,6 +52,7 @@ CompiledObjectKey, CompiledProject, CompiledSqlScenario, + CompilerDiagnostic, ) from sqlbuild.compiler.discovery.models import ( DiscoveredCheckFunction, @@ -69,6 +70,7 @@ ) from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan +from sqlbuild.compiler.refactoring.models import RefactorProject from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus from sqlbuild.cost.types import CostStatus from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult @@ -161,6 +163,7 @@ class BuildCommandRequest: allow_snapshot_full_refresh: bool = False allow_table_type_downgrade: bool = False allow_retention_decrease: bool = False + allow_missing_migration_origin: bool = False allow_snapshot_schema_change: bool = False concurrency: int | None = None select: tuple[str, ...] = () @@ -1052,6 +1055,29 @@ class TestCommandRequest: inspect: bool = False +@dataclass(frozen=True) +class RefactorCommandRequest: + """CLI inputs for one `sqb rename` or `sqb mv` invocation.""" + + command: str + target: str + new_name: str | None = None + destination: str | None = None + project_dir: Path | None = None + cascade: bool = False + dry_run: bool = False + json_output: bool = False + no_color: bool = False + + +@dataclass(frozen=True) +class RefactorCompile: + """One offline compile for a refactoring: its facts, or the errors that stopped it.""" + + project: RefactorProject | None + errors: tuple[CompilerDiagnostic, ...] + + @dataclass(frozen=True) class ScopeCommandRequest: """CLI inputs for one offline declaration-scope inspection.""" diff --git a/src/sqlbuild/cli/commands/types.py b/src/sqlbuild/cli/commands/types.py index 57203149cf..f9adb82b2d 100644 --- a/src/sqlbuild/cli/commands/types.py +++ b/src/sqlbuild/cli/commands/types.py @@ -53,6 +53,8 @@ class CliCommand(StrEnum): RULES = "rules" SCOPE = "scope" CONTRACT = "contract" + RENAME = "rename" + MV = "mv" class ScopeCommandHandler(Protocol): diff --git a/src/sqlbuild/cli/entry/models.py b/src/sqlbuild/cli/entry/models.py index c0b4ec4faa..371fafc465 100644 --- a/src/sqlbuild/cli/entry/models.py +++ b/src/sqlbuild/cli/entry/models.py @@ -78,3 +78,4 @@ class CliEntrypointHandlers: run_rules: Callable[..., int] run_scope: ScopeCommandHandler run_contract: Callable[..., int] | None = None + run_refactor: Callable[..., int] | None = None diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index b36a06510f..415ce91d6e 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -6,6 +6,7 @@ from dataclasses import fields from datetime import date, datetime from decimal import Decimal +from enum import StrEnum from pathlib import Path from typing import cast @@ -82,6 +83,7 @@ EventExportSeverity, FutureCursorAction, MicrobatchLimitAction, + MissingMigrationOriginPolicy, RetentionDecreasePolicy, TableType, TableTypeDowngradePolicy, @@ -1071,17 +1073,23 @@ def _optional_table_type( ) from exc -def _optional_table_type_downgrade_policy( - *, mapping: dict[str, object], key: str, label: str, file_path: Path -) -> TableTypeDowngradePolicy | None: +def _optional_target_policy[PolicyT: StrEnum]( + *, + mapping: dict[str, object], + key: str, + target_name: str, + file_path: Path, + policy: type[PolicyT], +) -> PolicyT | None: value: object | None = mapping.get(key) if value is None: return None try: - return TableTypeDowngradePolicy(value) + return policy(value) except (TypeError, ValueError) as exc: raise ProjectConfigError( - f"{file_path} {label} must be 'deny', 'require_confirmation', or 'allow'" + f"{file_path} targets.{target_name}.{key} must be 'deny', " + "'require_confirmation', or 'allow'" ) from exc @@ -1127,110 +1135,43 @@ def _target_retention_by_materialization( return loaded -def _optional_retention_decrease_policy( - *, mapping: dict[str, object], target_name: str, file_path: Path -) -> RetentionDecreasePolicy | None: - value: object | None = mapping.get("time_travel_retention_decrease") - if value is None: - return None - try: - return RetentionDecreasePolicy(value) - except (TypeError, ValueError) as exc: - raise ProjectConfigError( - f"{file_path} targets.{target_name}.time_travel_retention_decrease must be " - "'deny', 'require_confirmation', or 'allow'" - ) from exc +def _load_targets(*, payload: object, file_path: Path) -> dict[str, TargetConfig]: + return { + name: _project_target(local) + for name, local in _load_local_targets(payload=payload, file_path=file_path).items() + } -def _load_targets(*, payload: object, file_path: Path) -> dict[str, TargetConfig]: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="targets", file_path=file_path +def _project_target(local: LocalTargetConfig) -> TargetConfig: + defaults: TargetConfig = TargetConfig() + return TargetConfig( + connection=local.connection, + connection_name=local.connection_name, + vars=local.vars, + database=local.database, + schema=local.schema, + loader_schema=local.loader_schema, + defer_sources_to=local.defer_sources_to, + defer_clone_from=local.defer_clone_from, + clone=ClonePolicy( + allow_as_clone_origin=local.clone.allow_as_clone_origin is True, + allow_as_clone_destination=local.clone.allow_as_clone_destination is True, + ), + compile_cache=local.compile_cache, + time_travel_retention=local.time_travel_retention, + time_travel_retention_by_materialization=( + local.time_travel_retention_by_materialization or {} + ), + owns_time_travel_retention_namespace=local.owns_time_travel_retention_namespace is True, + default_table_type=local.default_table_type, + table_type_downgrade=local.table_type_downgrade or defaults.table_type_downgrade, + time_travel_retention_decrease=( + local.time_travel_retention_decrease or defaults.time_travel_retention_decrease + ), + missing_migration_origin=local.missing_migration_origin + or defaults.missing_migration_origin, + execution_limits=local.execution_limits, ) - targets: dict[str, TargetConfig] = {} - target_name: str - target_payload: object - for target_name, target_payload in mapping.items(): - target_mapping: dict[str, object] = _coerce_mapping( - payload=target_payload, - label=f"targets.{target_name}", - file_path=file_path, - ) - _validate_target_keys( - target_mapping=target_mapping, target_name=target_name, file_path=file_path - ) - clone_mapping: dict[str, object] = _coerce_mapping( - payload=target_mapping.get("clone"), - label=f"targets.{target_name}.clone", - file_path=file_path, - ) - _validate_clone_keys( - clone_mapping=clone_mapping, target_name=target_name, file_path=file_path - ) - execution_limits: ExecutionLimitsConfig = _load_execution_limits( - payload=target_mapping.get("execution_limits"), - label=f"targets.{target_name}.execution_limits", - file_path=file_path, - ) - connection, connection_name = _load_target_connection( - target_mapping=target_mapping, target_name=target_name, file_path=file_path - ) - targets[target_name] = TargetConfig( - connection=connection, - connection_name=connection_name, - vars=_load_string_mapping(payload=target_mapping.get("vars"), file_path=file_path), - database=_optional_str(payload=target_mapping, key="database"), - schema=_optional_str(payload=target_mapping, key="schema"), - loader_schema=_optional_str(payload=target_mapping, key="loader_schema"), - defer_sources_to=_optional_str(payload=target_mapping, key="defer_sources_to"), - defer_clone_from=_optional_str(payload=target_mapping, key="defer_clone_from"), - compile_cache=_optional_nullable_bool(mapping=target_mapping, key="compile_cache"), - time_travel_retention=_optional_target_retention_default( - mapping=target_mapping, target_name=target_name, file_path=file_path - ), - time_travel_retention_by_materialization=_target_retention_by_materialization( - mapping=target_mapping, target_name=target_name, file_path=file_path - ), - owns_time_travel_retention_namespace=_optional_bool( - mapping=target_mapping, - key="owns_time_travel_retention_namespace", - default=False, - ), - default_table_type=_optional_table_type( - mapping=target_mapping, - key="default_table_type", - label=f"targets.{target_name}.default_table_type", - file_path=file_path, - ), - table_type_downgrade=( - _optional_table_type_downgrade_policy( - mapping=target_mapping, - key="table_type_downgrade", - label=f"targets.{target_name}.table_type_downgrade", - file_path=file_path, - ) - or TableTypeDowngradePolicy.REQUIRE_CONFIRMATION - ), - time_travel_retention_decrease=( - _optional_retention_decrease_policy( - mapping=target_mapping, target_name=target_name, file_path=file_path - ) - or RetentionDecreasePolicy.DENY - ), - execution_limits=execution_limits, - clone=ClonePolicy( - allow_as_clone_origin=_optional_bool( - mapping=clone_mapping, - key="allow_as_clone_origin", - default=False, - ), - allow_as_clone_destination=_optional_bool( - mapping=clone_mapping, - key="allow_as_clone_destination", - default=False, - ), - ), - ) - return targets def _load_local_targets(*, payload: object, file_path: Path) -> dict[str, LocalTargetConfig]: @@ -1295,14 +1236,26 @@ def _load_local_targets(*, payload: object, file_path: Path) -> dict[str, LocalT label=f"targets.{target_name}.default_table_type", file_path=file_path, ), - table_type_downgrade=_optional_table_type_downgrade_policy( + table_type_downgrade=_optional_target_policy( mapping=target_mapping, key="table_type_downgrade", - label=f"targets.{target_name}.table_type_downgrade", + target_name=target_name, file_path=file_path, + policy=TableTypeDowngradePolicy, ), - time_travel_retention_decrease=_optional_retention_decrease_policy( - mapping=target_mapping, target_name=target_name, file_path=file_path + time_travel_retention_decrease=_optional_target_policy( + mapping=target_mapping, + key="time_travel_retention_decrease", + target_name=target_name, + file_path=file_path, + policy=RetentionDecreasePolicy, + ), + missing_migration_origin=_optional_target_policy( + mapping=target_mapping, + key="missing_migration_origin", + target_name=target_name, + file_path=file_path, + policy=MissingMigrationOriginPolicy, ), execution_limits=execution_limits, clone=LocalClonePolicy( @@ -1374,6 +1327,7 @@ def _validate_target_keys( "default_table_type", "table_type_downgrade", "time_travel_retention_decrease", + "missing_migration_origin", "clone", "execution_limits", } diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py index 2db0141960..9500d43388 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -21,6 +21,7 @@ rename_hints, ) from sqlbuild.compiler.planner._helpers.migrations.planning import ( + missing_origin_policy, planning_database, redirected_cursor_snapshots, ) @@ -32,6 +33,7 @@ from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, ) +from sqlbuild.compiler.planner.constants import MISSING_ORIGIN_OUTCOMES from sqlbuild.compiler.planner.models import ( ColumnMigrationPlanEntry, ColumnMigrationPlanning, @@ -47,6 +49,7 @@ ) from sqlbuild.compiler.planner.types import InputColumns, MaterializationType, WarningSeverity from sqlbuild.spec.contracts.main.get_config_str import get_config_str +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy _HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} @@ -141,6 +144,8 @@ def plan_column_migrations( entries.extend(model_entries) warnings.extend(model_warnings) hints.extend(_hints(candidate=candidate, entries=model_entries)) + origin_policy: MissingMigrationOriginPolicy = missing_origin_policy(runtime=runtime) + entries = [replace(entry, missing_origin_policy=origin_policy) for entry in entries] warnings.extend(warning for entry in entries if (warning := _entry_warning(entry))) overlaid: WarehouseSnapshot = _overlay_snapshot( snapshot=snapshot, @@ -399,7 +404,9 @@ def _overlay_snapshot( if not entries: return snapshot - blocked: frozenset[str] = frozenset(entry.model_name for entry in entries if entry.blocks_build) + blocked: frozenset[str] = frozenset( + entry.model_name for entry in entries if entry.blocks_build or entry.origin_missing + ) columns: dict[str, tuple[ColumnInfo, ...]] = dict(snapshot.existing_columns) fingerprints: dict[str, Fingerprint] = dict(snapshot.fingerprints.models) candidate: _ModelColumns @@ -521,8 +528,7 @@ def _entry_warning(entry: ColumnMigrationPlanEntry) -> PlanWarning | None: "M109", f"model '{entry.model_name}': {declaration}: column {entry.origin_column} does not " f"exist in {relation} and no recorded rename into {entry.destination_column} was " - "found; if the rename already happened elsewhere or is no longer needed, remove " - "migrate_from from the column", + f"found; {MISSING_ORIGIN_OUTCOMES[entry.missing_origin_policy]}", ), ColumnMigrationDecision.CONFLICT: ( "M110", @@ -546,7 +552,7 @@ def _entry_warning(entry: ColumnMigrationPlanEntry) -> PlanWarning | None: return None return PlanWarning( model_name=entry.model_name, - severity=WarningSeverity.ERROR, + severity=WarningSeverity.ERROR if entry.blocks_build else WarningSeverity.WARNING, message=coded[1], code=coded[0], ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py index 8982d3f1b9..e7dbb735c3 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py @@ -11,6 +11,7 @@ MigrationDiscovery, OldNameViewAction, ) +from sqlbuild.compiler.planner.constants import MANUAL_RENAME_HINT from sqlbuild.compiler.planner.models import ( ColumnMigrationPlanEntry, ModelMigrationPlanEntry, @@ -172,8 +173,10 @@ def _format_column_migrations( def _column_migration_text(*, entry: ColumnMigrationPlanEntry, style: CliStyle) -> str: rename: str = f"{entry.origin_column} -> {entry.destination_column}" decision: str - if entry.decision.blocks_build: + if entry.blocks_build: decision = style.error_strong(entry.decision.label) + elif entry.origin_missing: + decision = style.warning_strong(entry.decision.label) elif entry.decision == ColumnMigrationDecision.DONE: decision = style.muted(entry.decision.label) else: @@ -198,7 +201,7 @@ def _migration_lines( destination: str = entry.destination.qualified_name or entry.destination.name relation_move: str = f"{style.muted(f'{origin} ->')} {destination}" name: str = style.object_name(entry.model_name) - decision: str = _decision_text(decision=entry.decision, style=style) + decision: str = _decision_text(entry=entry, style=style) if entry.decision == MigrationDecision.RENAMED: decision = ( style.muted(MigrationDecision.DONE.label) @@ -240,6 +243,8 @@ def _migration_lines( ) if entry.discovery != MigrationDiscovery.MANUAL: rows.append(_property_row(label="discovery", value=entry.discovery.value, style=style)) + if entry.discovery == MigrationDiscovery.AUTOMATIC and entry.completed_at is None: + rows.append(_property_row(label="hint", value=MANUAL_RENAME_HINT, style=style)) if entry.completed_at is not None: rows.append( _property_row( @@ -250,8 +255,8 @@ def _migration_lines( ) if old_name is not None: rows.extend(old_name_rows(entry=old_name, style=style)) - blocking: bool = ( - entry.decision.blocks_build or entry.compatibility == MigrationCompatibility.INCOMPATIBLE + blocking: bool = entry.blocks_build or ( + entry.compatibility == MigrationCompatibility.INCOMPATIBLE ) finding_style: Callable[[str], str] = style.error if blocking else style.warning rows.extend(f" {finding_style(f'! {finding}')}" for finding in entry.compatibility_findings) @@ -270,9 +275,12 @@ def _property_row(*, label: str, value: str, style: CliStyle) -> str: return f" {style.muted(label)} {value}" -def _decision_text(*, decision: MigrationDecision, style: CliStyle) -> str: - if decision.blocks_build: +def _decision_text(*, entry: ModelMigrationPlanEntry, style: CliStyle) -> str: + decision: MigrationDecision = entry.decision + if entry.blocks_build and decision.blocks_build: return style.error_strong(decision.label) + if entry.origin_missing: + return style.warning_strong(decision.label) if decision in (MigrationDecision.SUPERSEDED_REPLACE, MigrationDecision.FORCED_REPLACE): return style.warning_strong(decision.label) if decision == MigrationDecision.DONE: diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index e759fa032b..b6af40b215 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -33,6 +33,7 @@ from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, ) +from sqlbuild.compiler.planner._helpers.planning.retention import effective_target_config from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( gather_redirected_cursor_snapshots, ) @@ -43,7 +44,9 @@ MigrationStateInspection, ) from sqlbuild.compiler.planner.constants import ( + HIDDEN_ORIGIN_REMEDY, MIGRATION_MODEL_NAME_METADATA_KEY, + MISSING_ORIGIN_OUTCOMES, QUALIFIED_RELATION_MAX_PARTS, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError @@ -66,7 +69,7 @@ from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity from sqlbuild.spec.contracts.main.get_config_str import get_config_str from sqlbuild.spec.contracts.models import SnapshotsConfig -from sqlbuild.spec.contracts.types import TableType +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, TableType _IDENTITY_MATERIALIZATIONS: frozenset[str] = frozenset( {MaterializationType.TABLE, MaterializationType.VIEW} @@ -177,6 +180,7 @@ def plan_model_migrations( if runtime.project_config is not None else SnapshotsConfig() ) + origin_policy: MissingMigrationOriginPolicy = missing_origin_policy(runtime=runtime) entries: list[ModelMigrationPlanEntry] = [] handovers: dict[str, Fingerprint | None] = {} request: ModelMigrationRequest @@ -195,7 +199,7 @@ def plan_model_migrations( state=state, snapshots_config=snapshots_config, ) - entries.append(entry) + entries.append(replace(entry, missing_origin_policy=origin_policy)) if applies: handovers[request.model.name] = _equivalent_definition( model=request.model, @@ -290,6 +294,12 @@ def planning_database(*, runtime: PlannerRuntime) -> str | None: ) +def missing_origin_policy(*, runtime: PlannerRuntime) -> MissingMigrationOriginPolicy: + """Return how the effective target treats a declared migration origin it cannot find.""" + + return effective_target_config(runtime=runtime).missing_migration_origin + + def project_schemas(*, runtime: PlannerRuntime) -> set[str]: """Return every destination schema owned by project models.""" @@ -755,14 +765,28 @@ def _entry_warning(entry: ModelMigrationPlanEntry) -> PlanWarning | None: ), code="M101", ) - if entry.decision == MigrationDecision.ORIGIN_MISSING: + if entry.origin_hidden: return PlanWarning( model_name=entry.model_name, severity=WarningSeverity.ERROR, message=( - f"model '{entry.model_name}': migrate_from origin {origin} does not exist and " - "no recorded migration into it was found; if the migration already happened " - "elsewhere or is no longer needed, remove migrate_from from the model header" + f"model '{entry.model_name}': migrate_from origin {origin} was built in this " + f"target but is not visible; {HIDDEN_ORIGIN_REMEDY}" + ), + code="M102", + ) + if entry.origin_missing: + return PlanWarning( + model_name=entry.model_name, + severity=( + WarningSeverity.ERROR + if entry.missing_origin_policy == MissingMigrationOriginPolicy.DENY + else WarningSeverity.WARNING + ), + message=( + f"model '{entry.model_name}': migrate_from origin {origin} does not exist in " + "this target and no recorded migration into it was found; " + + MISSING_ORIGIN_OUTCOMES[entry.missing_origin_policy] ), code="M102", ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py index 1b63c8c8c1..9842ce637e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py @@ -59,7 +59,7 @@ def plan_table_types( ) if runtime.adapter.adapter_name != BuiltinAdapter.SNOWFLAKE: return () - target: TargetConfig = _effective_target(runtime=runtime) + target: TargetConfig = effective_target_config(runtime=runtime) entries: list[TableTypePlanEntry] = [] for model in selected_models: if not MaterializationType.is_table_backed( @@ -140,7 +140,9 @@ def plan_retention( entries.extend( _plan_relation_retention(runtime=runtime, warehouse=warehouse, model=model) ) - decrease_policy: str = _effective_target(runtime=runtime).time_travel_retention_decrease.value + decrease_policy: str = effective_target_config( + runtime=runtime + ).time_travel_retention_decrease.value return tuple(replace(entry, decrease_policy=decrease_policy) for entry in entries) @@ -262,7 +264,7 @@ def _plan_relation_retention( def _plan_bigquery_retention( *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...] ) -> tuple[RetentionPlanEntry, ...]: - target: TargetConfig = _effective_target(runtime=runtime) + target: TargetConfig = effective_target_config(runtime=runtime) if not target.owns_time_travel_retention_namespace: raise PlannerInputError( "BigQuery managed time_travel_retention requires target " @@ -362,7 +364,9 @@ def _plan_bigquery_retention( return tuple(entries) -def _effective_target(*, runtime: PlannerRuntime) -> TargetConfig: +def effective_target_config(*, runtime: PlannerRuntime) -> TargetConfig: + """Return the resolved config of the target being planned, or defaults without one.""" + if ( runtime.project_config is None or runtime.local_config is None diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 991de00b28..59b474cb02 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -4,6 +4,7 @@ from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.cursor_algebra.types import BoundSentinel +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy PATH_SELECTOR_EXPLICIT_ROOT_ERROR: str = ( "path selectors require an explicit root: use 'models/' or 'python/'" @@ -83,3 +84,27 @@ SNAPSHOT_DEFAULT_VALID_TO_COLUMN: str = "valid_to" SOURCE_INPUT_FUNCTION: str = "__source" REF_INPUT_FUNCTION: str = "__ref" +MANUAL_RENAME_HINT: str = ( + "matched by unchanged definition; sqb rename model: rewrites references and " + "declares migrate_from so the move does not depend on discovery" +) +HIDDEN_ORIGIN_REMEDY: str = ( + "the connection may lack permission to see it, or it was dropped outside SQLBuild; restore " + "access to it, or remove migrate_from to build the model fresh deliberately " + "(missing_migration_origin does not apply)" +) +MISSING_ORIGIN_OUTCOMES: dict[MissingMigrationOriginPolicy, str] = { + MissingMigrationOriginPolicy.ALLOW: ( + "nothing is migrated and the build continues as if migrate_from were absent; remove " + "migrate_from once no target still needs it" + ), + MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION: ( + "building without it needs confirmation (missing_migration_origin = " + "require_confirmation); pass --allow-missing-migration-origin, or remove migrate_from if " + "it is no longer needed" + ), + MissingMigrationOriginPolicy.DENY: ( + "the build stops (missing_migration_origin = deny); if the migration already happened " + "elsewhere or is no longer needed, remove migrate_from" + ), +} diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 9a715e90db..8afa5db8ca 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -91,6 +91,7 @@ from sqlbuild.spec.contracts.types import ( FutureCursorAction, MicrobatchLimitAction, + MissingMigrationOriginPolicy, SourceWriteStrategy, ) from sqlbuild.sql_values.models import SqlValue @@ -931,6 +932,19 @@ class ModelMigrationPlanEntry: compatibility_findings: tuple[str, ...] = () message: str | None = None origin_tracked: bool = False + missing_origin_policy: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW + + @property + def origin_missing(self) -> bool: + """Return whether the declared origin was not found in this target.""" + + return self.decision == MigrationDecision.ORIGIN_MISSING + + @property + def origin_hidden(self) -> bool: + """Return whether the missing origin has build history in this target.""" + + return self.origin_missing and self.origin_tracked @property def storage_transition(self) -> str | None: @@ -951,6 +965,11 @@ def target_label(self) -> str: def blocks_build(self) -> bool: """Return whether this migration must stop a build before any execution.""" + if self.origin_missing: + return ( + self.origin_tracked + or self.missing_origin_policy == MissingMigrationOriginPolicy.DENY + ) return self.decision.blocks_build or ( self.decision.moves_data and self.compatibility == MigrationCompatibility.INCOMPATIBLE ) @@ -1010,11 +1029,20 @@ class ColumnMigrationPlanEntry: target_name: str | None completed_at: datetime | None = None message: str | None = None + missing_origin_policy: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW + + @property + def origin_missing(self) -> bool: + """Return whether the declared origin column was not found in this target.""" + + return self.decision == ColumnMigrationDecision.SOURCE_MISSING @property def blocks_build(self) -> bool: """Return whether this column migration must stop a build before any execution.""" + if self.origin_missing: + return self.missing_origin_policy == MissingMigrationOriginPolicy.DENY return self.decision.blocks_build diff --git a/src/sqlbuild/compiler/refactoring/__init__.py b/src/sqlbuild/compiler/refactoring/__init__.py new file mode 100644 index 0000000000..c2d3b86586 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/__init__.py @@ -0,0 +1 @@ +"""Compiler-driven project refactorings: model renames, moves, and column renames.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/__init__.py new file mode 100644 index 0000000000..355dcd1846 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/__init__.py @@ -0,0 +1 @@ +"""Internal refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/columns/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/columns/__init__.py new file mode 100644 index 0000000000..e0399ee685 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/columns/__init__.py @@ -0,0 +1 @@ +"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/columns/column_planning.py b/src/sqlbuild/compiler/refactoring/_helpers/columns/column_planning.py new file mode 100644 index 0000000000..bcd5e6e449 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/columns/column_planning.py @@ -0,0 +1,509 @@ +"""Walk a column rename from its owner model through every downstream reader.""" + +from __future__ import annotations + +from collections import deque + +from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.refactoring._helpers.columns.column_references import ( + analyze_column, + consumer_edits, + output_edits, + resource_columns, +) +from sqlbuild.compiler.refactoring._helpers.project.project_files import ( + authored_bodies, + project_sql_files, + python_string_locations, + yaml_files, +) +from sqlbuild.compiler.refactoring._helpers.renames.model_planning import ( + find_model, + needs_column_migration, + validate_identifier, +) +from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( + add_column_entry_edit, + column_config_edits, + column_entry_edits, + consumer_column_header_edits, + header_tokens, + unhandled_word_offsets, +) +from sqlbuild.compiler.refactoring._helpers.text.schema_edits import schema_column_edits +from sqlbuild.compiler.refactoring._helpers.text.sql_sites import ( + analysis_sql, + authored_offset, + authored_span, + model_body, +) +from sqlbuild.compiler.refactoring._helpers.text.text_edits import ( + manual_at, + merge_parts, + path_edits, +) +from sqlbuild.compiler.refactoring._helpers.text.yaml_edits import yaml_column_edits +from sqlbuild.compiler.refactoring.constants import ( + EXPECTED_FIXTURE_PREFIX, + FIXTURE_ROLES, + GENERIC_DIALECT, + MIGRATE_FROM_KEY, + REF_FIXTURE_PREFIX, + REF_KIND, + ROOT_SCOPE, +) +from sqlbuild.compiler.refactoring.exceptions import RefactorInputError +from sqlbuild.compiler.refactoring.models import ( + AnalysisSql, + AuthoredBody, + BodyContext, + BodyEdits, + ColumnFacts, + ColumnQuery, + ColumnRenameContext, + HeaderToken, + ManualLocation, + MigrationDeclaration, + ModelBody, + ProjectSqlFile, + RefactorParts, + RefactorProject, + RefactorRequest, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import SqlFileRole +from sqlbuild.spec.contracts.models import SchemaColumn + +type _ModelContext = tuple[BodyContext | None, str, tuple[ManualLocation, ...]] + + +def column_rename_context( + *, project: RefactorProject, request: RefactorRequest +) -> ColumnRenameContext: + """Validate a column rename and collect everything its walk reads.""" + + owner: CompiledModel = find_model(project=project.graph.project, name=request.model_name) + old: str = request.column_name or "" + new: str = request.new_name + validate_identifier(name=new, noun="column") + files: tuple[ProjectSqlFile, ...] = project_sql_files(discovered=project.discovered) + context: ColumnRenameContext = ColumnRenameContext( + project=project, + owner=owner, + dialect=project.graph.project.sql_analysis_dialect or GENERIC_DIALECT, + columns=resource_columns(project=project.graph.project), + contents={item.relative_path: item.contents for item in files}, + bodies=authored_bodies(discovered=project.discovered), + yaml_files=yaml_files(discovered=project.discovered), + schema_files=tuple(item for item in files if item.role == SqlFileRole.SCHEMA), + old=old, + new=new, + cascade=request.cascade, + ) + if old.lower() not in _output_names(context=context): + raise RefactorInputError( + f"model:{owner.name} has no output column '{old}'", + code="C956", + help="column names match case-insensitively; check the spelling", + ) + if old.lower() == new.lower(): + raise RefactorInputError(f"column {old} already has that name", code="C953") + return context + + +def column_rename_parts(*, context: ColumnRenameContext) -> RefactorParts: + """Return every edit of the rename; `cascaded` lists each model whose column is renamed.""" + + parts: list[RefactorParts] = [_collision(context=context), _owner_output(context=context)] + queue: deque[CompiledModel] = deque([context.owner]) + seen: set[str] = set() + while queue: + model: CompiledModel = queue.popleft() + if model.name in seen: + continue + seen.add(model.name) + consumers: RefactorParts + cascaded: tuple[CompiledModel, ...] + consumers, cascaded = _consumers(context=context, model=model) + parts.extend( + ( + RefactorParts(cascaded=(model.name,)), + _declarations(context=context, model=model), + consumers, + _authored_body_parts(context=context, model=model), + RefactorParts( + edits=( + *yaml_column_edits( + files=context.yaml_files, + model=model.name, + old=context.old, + new=context.new, + ), + *_schema_edits(context=context, model=model), + ) + ), + ) + ) + queue.extend(cascaded) + merged: RefactorParts = merge_parts(parts=tuple(parts)) + return merge_parts(parts=(merged, _python_locations(context=context, merged=merged))) + + +def _schema_edits( + *, context: ColumnRenameContext, model: CompiledModel +) -> tuple[tuple[str, TextEdit], ...]: + edits: list[tuple[str, TextEdit]] = [] + item: ProjectSqlFile + for item in context.schema_files: + edits.extend( + path_edits( + path=item.relative_path, + edits=schema_column_edits( + contents=item.contents, + upstream=model.name, + old=context.old, + new=context.new, + ), + ) + ) + return tuple(edits) + + +def _output_names(*, context: ColumnRenameContext) -> frozenset[str]: + return frozenset( + name.lower() for name in context.columns.get((REF_KIND, context.owner.name), ()) + ) + + +def _collision(*, context: ColumnRenameContext) -> RefactorParts: + if context.new.lower() not in _output_names(context=context): + return RefactorParts() + return RefactorParts( + blocking=( + ManualLocation( + path=context.owner.relative_path.as_posix(), + line=None, + column=None, + reason=f"model:{context.owner.name} already has a column named {context.new}", + ), + ) + ) + + +def _owner_output(*, context: ColumnRenameContext) -> RefactorParts: + body: BodyContext | None + compiled_sql: str + unmapped: tuple[ManualLocation, ...] + body, compiled_sql, unmapped = _model_context(context=context, model=context.owner) + if body is None: + return RefactorParts(manual=unmapped) + facts: ColumnFacts = analyze_column( + analysis=analysis_sql(text=compiled_sql, dialect=context.dialect), + dialect=context.dialect, + columns=context.columns, + query=ColumnQuery( + column=context.old, + target_tables=frozenset(), + output_scopes=frozenset({ROOT_SCOPE}), + ), + ) + result: BodyEdits = output_edits( + facts=facts, + context=body, + scopes=frozenset({ROOT_SCOPE}), + old=context.old, + new=context.new, + ) + missing: tuple[ManualLocation, ...] = ( + () + if result.edits or result.manual + else ( + ManualLocation( + path=body.path, + line=None, + column=None, + reason=f"cannot find where model:{context.owner.name} produces {context.old}", + ), + ) + ) + return RefactorParts( + edits=path_edits(path=body.path, edits=result.edits), manual=(*result.manual, *missing) + ) + + +def _declarations(*, context: ColumnRenameContext, model: CompiledModel) -> RefactorParts: + path: str = model.relative_path.as_posix() + declared: SchemaColumn | None = next( + ( + column + for column in (model.schema_entry.columns if model.schema_entry is not None else ()) + if column.name.lower() == context.old.lower() + ), + None, + ) + if declared is not None and declared.migrate_from is not None: + return RefactorParts( + blocking=( + ManualLocation( + path=path, + line=None, + column=None, + reason=( + f"column {context.old} of model:{model.name} still declares " + "migrate_from; build it on every target and remove migrate_from before " + "renaming it again" + ), + ), + ) + ) + shared: tuple[ManualLocation, ...] = ( + ( + ManualLocation( + path=declared.location.path.as_posix(), + line=declared.location.line, + column=declared.location.column, + reason=( + f"column {context.old} of model:{model.name} is declared in a shared schema" + ), + ), + ) + if declared is not None + and declared.location is not None + and declared.location.path.as_posix() != path + else () + ) + return merge_parts( + parts=(RefactorParts(manual=shared), _header_parts(context=context, model=model)) + ) + + +def _header_parts(*, context: ColumnRenameContext, model: CompiledModel) -> RefactorParts: + path: str = model.relative_path.as_posix() + contents: str = context.contents.get(path, "") + tokens: tuple[HeaderToken, ...] = header_tokens(contents=contents) or () + migrate: bool = needs_column_migration(model=model) + entry_edits: tuple[TextEdit, ...] | None = column_entry_edits( + contents=contents, tokens=tokens, old=context.old, new=context.new, migrate=migrate + ) + added: TextEdit | None = ( + add_column_entry_edit(contents=contents, tokens=tokens, new=context.new, old=context.old) + if entry_edits is None and migrate + else None + ) + edits: tuple[TextEdit, ...] = ( + *column_config_edits(contents=contents, tokens=tokens, old=context.old, new=context.new), + *(entry_edits or ()), + *((added,) if added is not None else ()), + ) + handled: frozenset[int] = frozenset(edit.start for edit in edits) + return RefactorParts( + edits=path_edits(path=path, edits=edits), + manual=tuple( + manual_at( + path=path, + text=contents, + offset=offset, + reason=f"the MODEL header mentions {context.old} here; update it by hand", + ) + for offset in unhandled_word_offsets( + contents=contents, tokens=tokens, word=context.old, handled=handled + ) + ), + migrations=( + MigrationDeclaration( + model_name=model.name, + declaration=f"{context.new} ({MIGRATE_FROM_KEY} {context.old})", + reason="keeps the column's history in place", + ), + ) + if migrate + else (), + ) + + +def _consumers( + *, context: ColumnRenameContext, model: CompiledModel +) -> tuple[RefactorParts, tuple[CompiledModel, ...]]: + parts: list[RefactorParts] = [] + cascaded: list[CompiledModel] = [] + consumer: CompiledModel + for consumer in context.project.graph.project.models: + if not any( + dep.resource_type == CompiledResourceType.MODEL and dep.name == model.name + for dep in consumer.deps + ): + continue + consumer_parts: RefactorParts + passes_through: bool + consumer_parts, passes_through = _consumer( + context=context, upstream=model, consumer=consumer + ) + parts.append(consumer_parts) + if passes_through: + cascaded.append(consumer) + return merge_parts(parts=tuple(parts)), tuple(cascaded) + + +def _consumer( + *, context: ColumnRenameContext, upstream: CompiledModel, consumer: CompiledModel +) -> tuple[RefactorParts, bool]: + body: BodyContext | None + compiled_sql: str + unmapped: tuple[ManualLocation, ...] + body, compiled_sql, unmapped = _model_context(context=context, model=consumer) + if body is None: + return RefactorParts(manual=unmapped), False + analysis: AnalysisSql = analysis_sql(text=compiled_sql, dialect=context.dialect) + facts: ColumnFacts = analyze_column( + analysis=analysis, + dialect=context.dialect, + columns=context.columns, + query=ColumnQuery( + column=context.old, + target_tables=analysis.tables.get((REF_KIND, upstream.name), frozenset()), + ), + ) + result: BodyEdits = consumer_edits( + facts=facts, + context=body, + old=context.old, + new=context.new, + cascade_root=context.cascade, + root_stars_pass=context.cascade, + ) + header: tuple[TextEdit, ...] = consumer_column_header_edits( + contents=body.contents, upstream=upstream.name, old=context.old, new=context.new + ) + return ( + RefactorParts( + edits=path_edits(path=body.path, edits=(*result.edits, *header)), + manual=result.manual, + ), + result.passes_through, + ) + + +def _authored_body_parts(*, context: ColumnRenameContext, model: CompiledModel) -> RefactorParts: + return merge_parts( + parts=tuple( + _authored_body(context=context, model=model, body=body) + for body in context.bodies + if model.name.lower() in body.text.lower() + ) + ) + + +def _authored_body( + *, context: ColumnRenameContext, model: CompiledModel, body: AuthoredBody +) -> RefactorParts: + lowered: str = body.text.lower() + analysis: AnalysisSql = analysis_sql(text=body.text, dialect=context.dialect) + targets: frozenset[str] = analysis.tables.get((REF_KIND, model.name), frozenset()) + fixture_scopes: frozenset[str] = ( + frozenset( + name + for name in ( + f"{EXPECTED_FIXTURE_PREFIX}{model.name}", + f"{REF_FIXTURE_PREFIX}{model.name}", + ) + if name.lower() in lowered + ) + if body.role in FIXTURE_ROLES + else frozenset() + ) + if not targets and not fixture_scopes: + return RefactorParts() + facts: ColumnFacts = analyze_column( + analysis=analysis, + dialect=context.dialect, + columns=context.columns, + query=ColumnQuery( + column=context.old, + target_tables=targets, + target_ctes=frozenset( + name for name in fixture_scopes if name.startswith(REF_FIXTURE_PREFIX) + ), + output_scopes=fixture_scopes, + ), + ) + located: BodyContext = BodyContext( + path=body.path, + contents=body.contents, + map_span=lambda start, end: (body.start + start, body.start + end), + locate=lambda offset: body.start + offset, + fallback_offset=body.start, + ) + outputs: BodyEdits = output_edits( + facts=facts, context=located, scopes=fixture_scopes, old=context.old, new=context.new + ) + consumers: BodyEdits = consumer_edits( + facts=facts, + context=located, + old=context.old, + new=context.new, + cascade_root=False, + root_stars_pass=True, + ) + return RefactorParts( + edits=path_edits(path=body.path, edits=(*outputs.edits, *consumers.edits)), + manual=(*outputs.manual, *consumers.manual), + ) + + +def _python_locations(*, context: ColumnRenameContext, merged: RefactorParts) -> RefactorParts: + edited: frozenset[str] = frozenset(path for path, _ in merged.edits) + return RefactorParts( + manual=tuple( + location + for location in python_string_locations( + project_dir=context.project.project_dir, + discovered=context.project.discovered, + names=(context.old,), + reason=( + f"Python SQL may read column {context.old} of {context.owner.name}; check it " + "by hand" + ), + context=context.owner.name, + ) + if location.path not in edited + ) + ) + + +def _model_context(*, context: ColumnRenameContext, model: CompiledModel) -> _ModelContext: + path: str = model.relative_path.as_posix() + contents: str | None = context.contents.get(path) + if contents is None: + return None, "", () + body: ModelBody | None = model_body( + project=context.project.graph.project, + project_dir=context.project.project_dir, + model=model, + contents=contents, + ) + if body is None: + return ( + None, + "", + ( + ManualLocation( + path=path, + line=None, + column=None, + reason="the compiled SQL of this model cannot be mapped back to the file", + ), + ), + ) + mapped: ModelBody = body + return ( + BodyContext( + path=path, + contents=contents, + map_span=lambda start, end: authored_span(body=mapped, start=start, end=end), + locate=lambda offset: authored_offset(body=mapped, offset=offset)[0], + fallback_offset=body.body_start, + ), + body.compiled_sql, + (), + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/columns/column_references.py b/src/sqlbuild/compiler/refactoring/_helpers/columns/column_references.py new file mode 100644 index 0000000000..ca6d6026a9 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/columns/column_references.py @@ -0,0 +1,398 @@ +"""Native column-reference facts for one SQL body, and the edits they imply.""" + +from __future__ import annotations + +import json +from typing import Any, cast + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.refactoring._helpers.text.text_edits import manual_at, text_edit +from sqlbuild.compiler.refactoring.constants import ( + CTE_SCOPE_PREFIX, + QUOTE_CHARACTERS, + REF_KIND, + ROOT_SCOPE, + SEED_KIND, + SOURCE_KIND, + UNKNOWN_COLUMN_TYPE, +) +from sqlbuild.compiler.refactoring.models import ( + AnalysisSql, + BodyContext, + BodyEdits, + ColumnFacts, + ColumnQuery, + ColumnReference, + ColumnSite, + ManualLocation, + OutputColumn, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import EditKind, NativeColumnReferences, ResourceColumns + + +def resource_columns(*, project: CompiledProject) -> ResourceColumns: + """Return the known output columns of every model, source, and seed.""" + + columns: ResourceColumns = {} + for model in project.models: + names: list[str] = [column.name for column in model.inferred_columns or ()] + names.extend( + column.name + for column in (model.schema_entry.columns if model.schema_entry is not None else ()) + if column.name not in names + ) + columns[(REF_KIND, model.name)] = tuple(names) + for source in project.sources: + columns[(SOURCE_KIND, source.name)] = tuple( + column.name for column in source.source_entry.columns + ) + for seed in project.seeds: + columns[(SEED_KIND, seed.name)] = tuple(column.name for column in seed.schema_entry.columns) + return columns + + +def analyze_column( + *, analysis: AnalysisSql, dialect: str, columns: ResourceColumns, query: ColumnQuery +) -> ColumnFacts: + """Resolve every use of one column of the target relations in one SQL body.""" + + request: dict[str, object] = { + "sql": analysis.sql, + "dialect": dialect, + "column": query.column, + "schema": {"strict": False, "tables": _schema_tables(analysis=analysis, columns=columns)}, + "target_tables": sorted(query.target_tables), + "target_ctes": sorted(query.target_ctes), + "output_ctes": sorted(query.output_scopes), + } + payload: dict[str, Any] = json.loads( + cast(NativeColumnReferences, _native).analyze_column_references_json(json.dumps(request)) + ) + return ColumnFacts( + parsed=bool(payload.get("parsed")), + references=tuple(_reference(item) for item in payload.get("references", ())), + stars=tuple(_site(item) for item in payload.get("stars", ())), + joins=tuple(_site(item) for item in payload.get("joins", ())), + unresolved=tuple(_site(item) for item in payload.get("unresolved", ())), + outputs=tuple(_output(item) for item in payload.get("outputs", ())), + star_outputs=tuple(str(item) for item in payload.get("starOutputs", ())), + ) + + +def _schema_tables(*, analysis: AnalysisSql, columns: ResourceColumns) -> list[dict[str, object]]: + tables: list[dict[str, object]] = [] + key: tuple[str, str] + placeholders: frozenset[str] + for key, placeholders in analysis.tables.items(): + known: list[dict[str, str]] = [ + {"name": name, "type": UNKNOWN_COLUMN_TYPE} for name in columns.get(key, ()) + ] + name: str + for name in sorted(placeholders): + table: dict[str, object] = {"name": name, "columns": known} + tables.append(table) + return tables + + +def _reference(item: dict[str, Any]) -> ColumnReference: + projection: dict[str, Any] | None = item.get("projection") + return ColumnReference( + start=int(item["start"]), + end=int(item["end"]), + name_start=int(item["nameStart"]), + name_end=int(item["nameEnd"]), + scope=str(item["scope"]), + projected=projection is not None, + alias=projection.get("alias") if projection is not None else None, + alias_start=projection.get("aliasStart") if projection is not None else None, + alias_end=projection.get("aliasEnd") if projection is not None else None, + ) + + +def _site(item: dict[str, Any]) -> ColumnSite: + return ColumnSite(scope=str(item["scope"]), start=item.get("start"), end=item.get("end")) + + +def _output(item: dict[str, Any]) -> OutputColumn: + return OutputColumn( + scope=str(item["scope"]), + start=item.get("start"), + end=item.get("end"), + alias_start=item.get("aliasStart"), + alias_end=item.get("aliasEnd"), + ) + + +def consumer_edits( + *, + facts: ColumnFacts, + context: BodyContext, + old: str, + new: str, + cascade_root: bool, + root_stars_pass: bool, +) -> BodyEdits: + """Rename every reference; keep consumer output names unless the root passes it on.""" + + if not facts.parsed: + return BodyEdits( + manual=(_manual(context=context, offset=None, reason="SQL could not be analysed"),) + ) + return combine_body_edits( + results=( + *( + _reference_edits( + reference=reference, + context=context, + old=old, + new=new, + cascade_root=cascade_root, + ) + for reference in facts.references + ), + *( + _star_edits(star=star, context=context, old=old, root_stars_pass=root_stars_pass) + for star in facts.stars + ), + *( + _finding( + context=context, + site=site, + reason=f"USING or NATURAL join on {old}; rewrite it as an ON condition", + ) + for site in facts.joins + ), + *( + _finding( + context=context, + site=site, + reason=f"cannot tell whether {old} here is the renamed column; qualify it", + ) + for site in facts.unresolved + ), + ) + ) + + +def combine_body_edits(*, results: tuple[BodyEdits, ...]) -> BodyEdits: + """Concatenate the edits of several analyses of one body.""" + + edits: list[TextEdit] = [] + manual: list[ManualLocation] = [] + result: BodyEdits + for result in results: + edits.extend(result.edits) + manual.extend(result.manual) + return BodyEdits( + edits=tuple(edits), + manual=tuple(manual), + passes_through=any(result.passes_through for result in results), + ) + + +def _reference_edits( + *, reference: ColumnReference, context: BodyContext, old: str, new: str, cascade_root: bool +) -> BodyEdits: + name_span: tuple[int, int] | None = context.map_span(reference.name_start, reference.name_end) + if name_span is None: + return BodyEdits( + manual=( + _manual( + context=context, + offset=context.locate(reference.name_start), + reason=f"column {old} is referenced in SQL a macro generates; update the " + "macro call by hand", + ), + ) + ) + authored_name: str = context.contents[name_span[0] : name_span[1]] + renamed: TextEdit = _rename_span(context=context, span=name_span, new=new) + if not reference.projected: + return BodyEdits(edits=(renamed,)) + same_name_alias: bool = reference.alias is not None and reference.alias.lower() == old.lower() + if ( + reference.scope == ROOT_SCOPE + and cascade_root + and (reference.alias is None or same_name_alias) + ): + alias_span: tuple[int, int] | None = ( + context.map_span(reference.alias_start, reference.alias_end) + if same_name_alias + and reference.alias_start is not None + and reference.alias_end is not None + else None + ) + alias_edits: tuple[TextEdit, ...] = ( + (_rename_span(context=context, span=alias_span, new=new),) + if alias_span is not None + else () + ) + return BodyEdits(edits=(renamed, *alias_edits), passes_through=True) + if reference.alias is not None: + return BodyEdits(edits=(renamed,)) + end_span: tuple[int, int] | None = context.map_span(reference.start, reference.end) + if end_span is None: + return BodyEdits( + manual=( + _manual( + context=context, + offset=name_span[0], + reason=f"column {old} is projected inside a macro call", + ), + ) + ) + return BodyEdits( + edits=( + renamed, + text_edit( + text=context.contents, + start=end_span[1], + end=end_span[1], + replacement=f" AS {authored_name}", + kind=EditKind.COLUMN, + before="", + after=f"AS {authored_name}", + ), + ) + ) + + +def _star_edits( + *, star: ColumnSite, context: BodyContext, old: str, root_stars_pass: bool +) -> BodyEdits: + if star.scope.startswith(CTE_SCOPE_PREFIX): + return BodyEdits() + if star.scope == ROOT_SCOPE and root_stars_pass: + return BodyEdits(passes_through=True) + return _finding( + context=context, + site=star, + reason=( + f"SELECT * passes {old} through, so this model's output column would be renamed " + "too; rerun with --cascade or list the columns" + if star.scope == ROOT_SCOPE + else f"SELECT * in a subquery passes {old} through; list the columns" + ), + ) + + +def _finding(*, context: BodyContext, site: ColumnSite, reason: str) -> BodyEdits: + return BodyEdits( + manual=( + _manual( + context=context, offset=_site_offset(context=context, site=site), reason=reason + ), + ) + ) + + +def _rename_span(*, context: BodyContext, span: tuple[int, int], new: str) -> TextEdit: + return text_edit( + text=context.contents, + start=span[0], + end=span[1], + replacement=_respell(authored=context.contents[span[0] : span[1]], name=new), + kind=EditKind.COLUMN, + ) + + +def output_edits( + *, facts: ColumnFacts, context: BodyContext, scopes: frozenset[str], old: str, new: str +) -> BodyEdits: + """Rename the projection that names the column in the given output scopes.""" + + edits: list[TextEdit] = [] + manual: list[ManualLocation] = [] + found: set[str] = set() + output: OutputColumn + for output in facts.outputs: + found.add(output.scope) + if output.alias_start is not None and output.alias_end is not None: + alias_span: tuple[int, int] | None = context.map_span( + output.alias_start, output.alias_end + ) + if alias_span is None: + manual.append( + _manual( + context=context, + offset=context.locate(output.alias_start), + reason=f"column {old} is named inside a macro call", + ) + ) + continue + edits.append( + text_edit( + text=context.contents, + start=alias_span[0], + end=alias_span[1], + replacement=_respell( + authored=context.contents[alias_span[0] : alias_span[1]], name=new + ), + kind=EditKind.COLUMN, + ) + ) + continue + if output.start is None or output.end is None: + continue + column_span: tuple[int, int] | None = context.map_span(output.start, output.end) + if column_span is None: + manual.append( + _manual( + context=context, + offset=context.locate(output.start), + reason=f"column {old} is produced inside a macro call", + ) + ) + continue + edits.append( + text_edit( + text=context.contents, + start=column_span[1], + end=column_span[1], + replacement=f" AS {new}", + kind=EditKind.COLUMN, + before="", + after=f"AS {new}", + ) + ) + missing: frozenset[str] = frozenset( + scope for scope in scopes if _scope_role(scope) not in found + ) + scope: str + for scope in sorted(missing): + if _scope_role(scope) in facts.star_outputs: + manual.append( + _manual( + context=context, + offset=None, + reason=f"{old} comes from SELECT * in {scope}; list the columns explicitly", + ) + ) + return BodyEdits(edits=tuple(edits), manual=tuple(manual)) + + +def _scope_role(scope: str) -> str: + return scope if scope == ROOT_SCOPE else f"{CTE_SCOPE_PREFIX}{scope}" + + +def _respell(*, authored: str, name: str) -> str: + if authored and authored[0] in QUOTE_CHARACTERS: + return f"{authored[0]}{name}{QUOTE_CHARACTERS[authored[0]]}" + return name + + +def _site_offset(*, context: BodyContext, site: ColumnSite) -> int | None: + if site.start is None: + return None + return context.locate(site.start) + + +def _manual(*, context: BodyContext, offset: int | None, reason: str) -> ManualLocation: + return manual_at( + path=context.path, + text=context.contents, + offset=context.fallback_offset if offset is None else offset, + reason=reason, + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/project/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/project/__init__.py new file mode 100644 index 0000000000..e0399ee685 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/project/__init__.py @@ -0,0 +1 @@ +"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/project/project_files.py b/src/sqlbuild/compiler/refactoring/_helpers/project/project_files.py new file mode 100644 index 0000000000..38f93ceca6 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/project/project_files.py @@ -0,0 +1,199 @@ +"""Authored project files and SQL bodies a refactoring reads and edits.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +from sqlbuild.compiler.discovery.models import ( + DiscoveredAuditBlock, + DiscoveredProjectInputs, + DiscoveredSqlTestBlock, +) +from sqlbuild.compiler.refactoring._helpers.text.text_edits import whole_word_offsets +from sqlbuild.compiler.refactoring.constants import SQL_LIKE_PATTERN +from sqlbuild.compiler.refactoring.models import AuthoredBody, ManualLocation, ProjectSqlFile +from sqlbuild.compiler.refactoring.types import SqlFileRole + +type _AuthoredFile = tuple[SqlFileRole, Path, str, tuple[str, ...]] + + +def project_sql_files(*, discovered: DiscoveredProjectInputs) -> tuple[ProjectSqlFile, ...]: + """Return every discovered SQL file with its authored contents.""" + + files: dict[str, ProjectSqlFile] = {} + role: SqlFileRole + relative_path: Path + contents: str + for role, relative_path, contents, _ in ( + *_model_files(discovered=discovered), + *_authored_files(discovered=discovered), + ): + relative: str = relative_path.as_posix() + files.setdefault( + relative, ProjectSqlFile(relative_path=relative, contents=contents, role=role) + ) + return tuple(files[path] for path in sorted(files)) + + +def yaml_files(*, discovered: DiscoveredProjectInputs) -> tuple[ProjectSqlFile, ...]: + """Return every source and seed YAML declaration file.""" + + files: dict[str, str] = { + item.relative_path.as_posix(): item.contents + for item in (*discovered.source_files, *discovered.schema_files) + } + return tuple( + ProjectSqlFile(relative_path=path, contents=files[path], role=SqlFileRole.YAML) + for path in sorted(files) + ) + + +def authored_bodies(*, discovered: DiscoveredProjectInputs) -> tuple[AuthoredBody, ...]: + """Locate every non-model SQL body in its file, in file order.""" + + bodies: list[AuthoredBody] = [] + role: SqlFileRole + relative_path: Path + contents: str + texts: tuple[str, ...] + for role, relative_path, contents, texts in _authored_files(discovered=discovered): + cursor: int = 0 + text: str + for text in texts: + start: int = contents.find(text, cursor) if text.strip() else -1 + if start < 0: + continue + cursor = start + len(text) + bodies.append( + AuthoredBody( + path=relative_path.as_posix(), + role=role, + contents=contents, + start=start, + text=text, + ) + ) + return tuple(bodies) + + +def python_string_locations( + *, + project_dir: Path, + discovered: DiscoveredProjectInputs, + names: tuple[str, ...], + reason: str, + context: str | None = None, +) -> tuple[ManualLocation, ...]: + """Flag Python strings that name a model or column; with context, only SQL naming it too.""" + + locations: list[ManualLocation] = [] + relative_path: Path + for relative_path in _python_paths(discovered=discovered): + locations.extend( + _string_locations( + project_dir=project_dir, + relative_path=relative_path, + names=names, + reason=reason, + context=context, + ) + ) + return tuple(locations) + + +def _model_files(*, discovered: DiscoveredProjectInputs) -> tuple[_AuthoredFile, ...]: + return tuple( + (SqlFileRole.MODEL, item.relative_path, item.contents, ()) + for item in discovered.model_files + ) + + +def _authored_files(*, discovered: DiscoveredProjectInputs) -> tuple[_AuthoredFile, ...]: + files: list[_AuthoredFile] = [] + files.extend( + (SqlFileRole.TEST, item.relative_path, item.contents, _test_bodies(item.blocks)) + for item in discovered.test_files + ) + files.extend( + (SqlFileRole.SCENARIO, item.relative_path, item.contents, (item.sql_body,)) + for item in discovered.scenario_files + ) + files.extend( + (SqlFileRole.AUDIT, item.relative_path, item.contents, _audit_bodies(item.blocks)) + for item in discovered.audit_files + ) + files.extend( + (SqlFileRole.HOOK, item.relative_path, item.contents, (item.sql_body,)) + for item in discovered.sql_hook_files + ) + files.extend( + (SqlFileRole.FUNCTION, item.relative_path, item.contents, (item.body_sql,)) + for item in discovered.sql_function_files + ) + files.extend( + (SqlFileRole.SCHEMA, item.relative_path, item.contents, ()) + for item in discovered.model_schema_files + ) + return tuple(files) + + +def _test_bodies(blocks: tuple[DiscoveredSqlTestBlock, ...]) -> tuple[str, ...]: + return tuple(block.sql_body for block in blocks) + + +def _audit_bodies(blocks: tuple[DiscoveredAuditBlock, ...]) -> tuple[str, ...]: + return tuple(block.sql_body for block in blocks) + + +def _python_paths(*, discovered: DiscoveredProjectInputs) -> tuple[Path, ...]: + paths: set[Path] = set() + paths.update(item.relative_path for item in discovered.loader_functions) + paths.update(item.relative_path for item in discovered.task_functions) + paths.update(item.relative_path for item in discovered.asset_functions) + paths.update(item.relative_path for item in discovered.check_functions) + paths.update(item.relative_path for item in discovered.hook_functions) + paths.update(item.relative_path for item in discovered.audit_factories) + paths.update(item.relative_path for item in discovered.python_function_files) + paths.update(item.relative_path for item in discovered.materialization_files) + return tuple(sorted(paths)) + + +def _string_locations( + *, + project_dir: Path, + relative_path: Path, + names: tuple[str, ...], + reason: str, + context: str | None, +) -> tuple[ManualLocation, ...]: + try: + tree: ast.Module = ast.parse((project_dir / relative_path).read_text(encoding="utf-8")) + except (OSError, SyntaxError, UnicodeError): + return () + locations: list[ManualLocation] = [] + node: ast.AST + for node in ast.walk(tree): + if not isinstance(node, ast.Constant) or not isinstance(node.value, str): + continue + if _flags(value=node.value, names=names, context=context): + locations.append( + ManualLocation( + path=relative_path.as_posix(), + line=node.lineno, + column=node.col_offset + 1, + reason=reason, + ) + ) + return tuple(locations) + + +def _flags(*, value: str, names: tuple[str, ...], context: str | None) -> bool: + sql_like: bool = bool(SQL_LIKE_PATTERN.search(value)) + if context is not None and not (sql_like and whole_word_offsets(text=value, word=context)): + return False + return any( + value.strip().lower() == name.lower() + or (sql_like and bool(whole_word_offsets(text=value, word=name))) + for name in names + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py b/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py new file mode 100644 index 0000000000..824d494760 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py @@ -0,0 +1,189 @@ +"""Stage refactoring edits in a scratch copy, then commit them with rollback.""" + +from __future__ import annotations + +import os +import shutil +from pathlib import Path, PurePosixPath + +from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring.constants import ( + COPIED_PROJECT_SUFFIXES, + HIDDEN_PREFIX, + IGNORED_PROJECT_DIRECTORIES, + PYCACHE_DIRECTORY, + TEXT_ENCODING, +) +from sqlbuild.compiler.refactoring.exceptions import RefactorWriteError +from sqlbuild.compiler.refactoring.models import FileChange +from sqlbuild.lint.main.write_atomically import write_atomically + + +def copy_project_inputs(*, project_dir: Path, staging_dir: Path) -> int: + """Copy every file compilation can read, skipping build output and hidden directories.""" + + copied: int = 0 + root: str + directories: list[str] + names: list[str] + for root, directories, names in os.walk(project_dir): + current: Path = Path(root) + directories[:] = sorted( + name + for name in directories + if not name.startswith(HIDDEN_PREFIX) + and not (current == project_dir and name in IGNORED_PROJECT_DIRECTORIES) + and name != PYCACHE_DIRECTORY + ) + name: str + for name in names: + source: Path = current / name + if source.suffix not in COPIED_PROJECT_SUFFIXES or name.startswith(HIDDEN_PREFIX): + continue + destination: Path = staging_dir / source.relative_to(project_dir) + destination.parent.mkdir(parents=True, exist_ok=True) + _ = shutil.copyfile(source, destination) + copied += 1 + return copied + + +def edited_contents( + *, originals: dict[str, str], changes: tuple[FileChange, ...] +) -> dict[str, str]: + """Return the new text of every changed file, keyed by its final path.""" + + return { + change.path: apply_text_edits(text=originals[change.original_path], edits=change.edits) + for change in changes + } + + +def read_originals(*, project_dir: Path, changes: tuple[FileChange, ...]) -> dict[str, str]: + """Read the current text of every file a plan changes.""" + + return { + change.original_path: (project_dir / PurePosixPath(change.original_path)).read_text( + encoding=TEXT_ENCODING + ) + for change in changes + } + + +def write_staged_changes( + *, staging_dir: Path, originals: dict[str, str], changes: tuple[FileChange, ...] +) -> tuple[Path, ...]: + """Apply a plan to the scratch copy and return the written paths.""" + + contents: dict[str, str] = edited_contents(originals=originals, changes=changes) + change: FileChange + for change in changes: + if change.moved: + (staging_dir / PurePosixPath(change.original_path)).unlink(missing_ok=True) + written: list[Path] = [] + for change in changes: + target: Path = staging_dir / PurePosixPath(change.path) + target.parent.mkdir(parents=True, exist_ok=True) + _ = target.write_text(contents[change.path], encoding=TEXT_ENCODING) + written.append(target) + prune_emptied_directories(root=staging_dir, changes=changes) + return tuple(written) + + +def commit_changes( + *, project_dir: Path, originals: dict[str, str], changes: tuple[FileChange, ...] +) -> tuple[Path, ...]: + """Write a verified plan into the project, restoring every file if any write fails.""" + + current: dict[str, str] = read_originals(project_dir=project_dir, changes=changes) + if current != originals: + changed: list[str] = sorted( + path for path in originals if current.get(path) != originals[path] + ) + raise RefactorWriteError( + f"files changed while the refactoring ran: {', '.join(changed)}", + help="nothing was written; run the command again", + ) + contents: dict[str, str] = edited_contents(originals=originals, changes=changes) + created_directories: list[Path] = [] + written: list[Path] = [] + try: + change: FileChange + for change in changes: + target: Path = project_dir / PurePosixPath(change.path) + created_directories.extend(_missing_directories(path=target.parent)) + target.parent.mkdir(parents=True, exist_ok=True) + written.append(target) + _write_file( + path=target, + contents=contents[change.path], + mode_source=project_dir / PurePosixPath(change.original_path), + ) + for change in changes: + if change.moved: + (project_dir / PurePosixPath(change.original_path)).unlink() + except BaseException: + _rollback( + project_dir=project_dir, + originals=originals, + written=written, + created_directories=created_directories, + ) + raise + prune_emptied_directories(root=project_dir, changes=changes) + return tuple(written) + + +def prune_emptied_directories(*, root: Path, changes: tuple[FileChange, ...]) -> None: + """Remove folders a move left empty, walking up to the project root.""" + + change: FileChange + for change in changes: + if not change.moved: + continue + directory: Path = (root / PurePosixPath(change.original_path)).parent + while directory != root and directory.is_dir() and not any(directory.iterdir()): + directory.rmdir() + directory = directory.parent + + +def _rollback( + *, + project_dir: Path, + originals: dict[str, str], + written: list[Path], + created_directories: list[Path], +) -> None: + original_paths: frozenset[Path] = frozenset( + project_dir / PurePosixPath(path) for path in originals + ) + path: Path + for path in written: + if path not in original_paths: + path.unlink(missing_ok=True) + relative: str + text: str + for relative, text in originals.items(): + restored: Path = project_dir / PurePosixPath(relative) + if restored.exists(): + write_atomically(path=restored, contents=text) + else: + _ = restored.write_text(text, encoding=TEXT_ENCODING) + directory: Path + for directory in reversed(created_directories): + if directory.is_dir() and not any(directory.iterdir()): + directory.rmdir() + + +def _write_file(*, path: Path, contents: str, mode_source: Path) -> None: + if not path.exists(): + _ = shutil.copy(mode_source, path) + write_atomically(path=path, contents=contents) + + +def _missing_directories(*, path: Path) -> list[Path]: + missing: list[Path] = [] + current: Path = path + while not current.exists(): + missing.append(current) + current = current.parent + return list(reversed(missing)) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/renames/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/renames/__init__.py new file mode 100644 index 0000000000..e0399ee685 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/renames/__init__.py @@ -0,0 +1 @@ +"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/renames/model_planning.py b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_planning.py new file mode 100644 index 0000000000..b566d7490d --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_planning.py @@ -0,0 +1,407 @@ +"""Resolve a model rename or move, find what blocks it, and decide its migrations.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path, PurePosixPath + +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.refactoring._helpers.project.project_files import ( + project_sql_files, + python_string_locations, + yaml_files, +) +from sqlbuild.compiler.refactoring._helpers.renames.model_references import ( + macro_reference_locations, + model_reference_edits, +) +from sqlbuild.compiler.refactoring._helpers.text.header_edits import header_tokens, is_value +from sqlbuild.compiler.refactoring._helpers.text.text_edits import manual_at +from sqlbuild.compiler.refactoring._helpers.text.yaml_edits import yaml_model_edits +from sqlbuild.compiler.refactoring.constants import ( + GENERIC_DIALECT, + HISTORY_MATERIALIZATIONS, + IDENTIFIER_PATTERN, + MATERIALIZED_KEY, + MIGRATABLE_MATERIALIZATIONS, + MIGRATE_FROM_KEY, + MODEL_KIND_PREFIX, + PATH_SEPARATOR, + SQL_SUFFIX, +) +from sqlbuild.compiler.refactoring.exceptions import RefactorInputError +from sqlbuild.compiler.refactoring.models import ( + HeaderToken, + ManualLocation, + ModelMigrationDecision, + ModelTarget, + ProjectSqlFile, + RefactorParts, + RefactorProject, + RefactorRequest, +) +from sqlbuild.compiler.refactoring.types import RefactorOperation +from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup +from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move +from sqlbuild.compiler.scopes.main.relocate_declarations_for_move import ( + relocate_declarations_for_move, +) +from sqlbuild.compiler.scopes.models import ( + DeclarationIdentity, + DeclarationRecord, + MovePreview, + ResourceIdentity, + ScopeDiagnostic, + ScopeIndex, +) +from sqlbuild.compiler.scopes.types import ResourceKind +from sqlbuild.spec.contracts.main.get_config_str import get_config_str + + +def find_model(*, project: CompiledProject, name: str) -> CompiledModel: + """Return the compiled model a kind-qualified target names.""" + + model: CompiledModel | None = next((item for item in project.models if item.name == name), None) + if model is None: + raise RefactorInputError( + f"{MODEL_KIND_PREFIX}{name} is not a model in this project", + code="C951", + help="list models with sqb dag --json or sqb scope --browse models", + ) + return model + + +def validate_identifier(*, name: str, noun: str) -> None: + """Refuse a new name that is not a plain SQL identifier.""" + + if not IDENTIFIER_PATTERN.match(name): + raise RefactorInputError( + f"'{name}' is not a valid {noun} name", + code="C952", + help="use letters, digits, and underscores, starting with a letter or underscore", + ) + + +def model_target(*, project: RefactorProject, request: RefactorRequest) -> ModelTarget: + """Resolve the model, its new name, and its destination file.""" + + model: CompiledModel = find_model(project=project.graph.project, name=request.model_name) + source_path: str = model.relative_path.as_posix() + destination: str = ( + _resolve_destination( + project_dir=project.project_dir, + raw=request.destination or "", + file_name=model.relative_path.name, + ) + if request.operation == RefactorOperation.MOVE_MODEL + else PurePosixPath(source_path).with_name(f"{request.new_name}{SQL_SUFFIX}").as_posix() + ) + new: str = ( + PurePosixPath(destination).stem + if request.operation == RefactorOperation.MOVE_MODEL + else request.new_name + ) + validate_identifier(name=new, noun="model") + if new == model.name and destination == source_path: + raise RefactorInputError( + f"model:{model.name} already has that name and location", code="C953" + ) + return ModelTarget( + model=model, + request=replace(request, new_name=new, destination=destination), + source_path=source_path, + ) + + +def model_parts(*, project: RefactorProject, target: ModelTarget) -> RefactorParts: + """Return every edit, manual location, and blocker of one model rename or move.""" + + files: tuple[ProjectSqlFile, ...] = project_sql_files(discovered=project.discovered) + dialect: str = project.graph.project.sql_analysis_dialect or GENERIC_DIALECT + old: str = target.model.name + new: str = target.request.new_name + moves: tuple[tuple[str, str], ...] + move_blockers: tuple[ManualLocation, ...] + moves, move_blockers = _declaration_moves(project=project, target=target) + return RefactorParts( + edits=( + *model_reference_edits(files=files, old=old, new=new, dialect=dialect), + *yaml_model_edits(files=yaml_files(discovered=project.discovered), old=old, new=new), + ) + if new != old + else (), + manual=( + *macro_reference_locations(project=project, files=files, old=old, dialect=dialect), + *python_string_locations( + project_dir=project.project_dir, + discovered=project.discovered, + names=(old,), + reason=f"Python code names model {old}; update it by hand", + ), + ), + blocking=( + *_collisions(project=project, target=target), + *move_blockers, + *_pending_migration(files=files, target=target), + ), + moves=moves, + ) + + +def _resolve_destination(*, project_dir: Path, raw: str, file_name: str) -> str: + path: Path = Path(raw) + absolute: Path = path if path.is_absolute() else project_dir / path + if raw.endswith(PATH_SEPARATOR) or absolute.is_dir(): + absolute = absolute / file_name + resolved: Path = absolute.resolve() + if not resolved.is_relative_to(project_dir): + raise RefactorInputError(f"destination '{raw}' is outside the project", code="C955") + if resolved.suffix != SQL_SUFFIX: + raise RefactorInputError( + f"destination '{raw}' must be a .sql file or a folder ending in /", code="C955" + ) + return resolved.relative_to(project_dir).as_posix() + + +def _collisions(*, project: RefactorProject, target: ModelTarget) -> tuple[ManualLocation, ...]: + new: str = target.request.new_name + found: list[ManualLocation] = [ + ManualLocation( + path=other.relative_path.as_posix(), + line=None, + column=None, + reason=f"model:{new} already exists", + ) + for other in project.graph.project.models + if other.name.lower() == new.lower() and other.name != target.model.name + ] + if ( + target.destination != target.source_path + and (project.project_dir / PurePosixPath(target.destination)).exists() + ): + found.append( + ManualLocation( + path=target.destination, + line=None, + column=None, + reason="destination file already exists", + ) + ) + return tuple(found) + + +def _declaration_moves( + *, project: RefactorProject, target: ModelTarget +) -> tuple[tuple[tuple[str, str], ...], tuple[ManualLocation, ...]]: + """Move the declarations the model uses to where placement requires them afterwards.""" + + if PurePosixPath(target.destination).parent == PurePosixPath(target.source_path).parent: + return (), () + index: ScopeIndex = project.graph.project.scope_index + move: MovePreview | None + diagnostics: tuple[ScopeDiagnostic, ...] + move, diagnostics = preview_scope_move( + lookup=build_scope_lookup(index=index), + resource=f"{MODEL_KIND_PREFIX}{target.model.name}", + destination=target.destination, + ) + relocated: tuple[DeclarationRecord, ...] | None = ( + None + if move is None + else relocate_declarations_for_move( + index=index, + resource=ResourceIdentity(kind=ResourceKind.MODEL, name=target.model.name), + destination=target.destination, + ) + ) + if move is None or relocated is None: + return (), tuple( + ManualLocation(path=target.destination, line=None, column=None, reason=item.message) + for item in diagnostics + ) or ( + ManualLocation( + path=target.destination, + line=None, + column=None, + reason="declaration placement at the destination could not be worked out", + ), + ) + return _file_moves(project=project, index=index, relocated=relocated) + + +def _file_moves( + *, project: RefactorProject, index: ScopeIndex, relocated: tuple[DeclarationRecord, ...] +) -> tuple[tuple[tuple[str, str], ...], tuple[ManualLocation, ...]]: + destinations: dict[str, str] = {} + blocking: list[ManualLocation] = [] + record: DeclarationRecord + for record in sorted(relocated, key=lambda item: item.identity): + original: DeclarationRecord = next( + item for item in index.declarations if item.identity == record.identity + ) + label: str = _label(record.identity) + if not (project.project_dir / PurePosixPath(original.path)).is_file(): + blocking.append( + _declaration_blocker( + record=original, + reason=f"{label} must move to {record.path}, but it is not an authored " + "project file", + ) + ) + continue + planned: str | None = destinations.setdefault(original.path, record.path) + if planned != record.path: + blocking.append( + _declaration_blocker( + record=original, + reason=f"{original.path} holds declarations that must move to different " + f"folders ({planned}, {record.path}); split the file first", + ) + ) + moving: frozenset[DeclarationIdentity] = frozenset(item.identity for item in relocated) + staying: DeclarationRecord + for staying in index.declarations: + if staying.path in destinations and staying.identity not in moving: + blocking.append( + _declaration_blocker( + record=staying, + reason=f"{destinations[staying.path]} would take " + f"{_label(staying.identity)} along, which must " + f"stay in {staying.path}; split the file first", + ) + ) + new_path: str + for new_path in sorted(set(destinations.values())): + if (project.project_dir / PurePosixPath(new_path)).exists(): + blocking.append( + ManualLocation( + path=new_path, + line=None, + column=None, + reason="a declaration must move here, but the file already exists", + ) + ) + return tuple(sorted(destinations.items())), tuple(blocking) + + +def _label(identity: DeclarationIdentity) -> str: + return f"{identity.kind.value}:{identity.name}" + + +def _declaration_blocker(*, record: DeclarationRecord, reason: str) -> ManualLocation: + return ManualLocation(path=record.path, line=record.line, column=record.column, reason=reason) + + +def _pending_migration( + *, files: tuple[ProjectSqlFile, ...], target: ModelTarget +) -> tuple[ManualLocation, ...]: + if MIGRATE_FROM_KEY not in target.model.config.values: + return () + contents: str = next( + (item.contents for item in files if item.relative_path == target.source_path), "" + ) + token: HeaderToken | None = next( + ( + item + for item in header_tokens(contents=contents) or () + if item.depth == 0 and is_value(token=item, value=MIGRATE_FROM_KEY) + ), + None, + ) + return ( + manual_at( + path=target.source_path, + text=contents, + offset=token.start if token is not None else None, + reason=( + f"model:{target.model.name} still declares migrate_from; build it on every " + "target and remove migrate_from before renaming it again" + ), + ), + ) + + +def decide_model_migration( + *, before: CompiledProject, after: CompiledProject | None, old: str, new: str +) -> ModelMigrationDecision: + """Declare migrate_from whenever the relation moves, unless migrations cannot follow it.""" + + old_model: CompiledModel | None = _model(project=before, name=old) + new_model: CompiledModel | None = None if after is None else _model(project=after, name=new) + if old_model is None: + return ModelMigrationDecision(needed=False, reason="model not compiled") + materialized: str | None = get_config_str( + values=(new_model or old_model).config.values, key=MATERIALIZED_KEY + ) + if materialized not in MIGRATABLE_MATERIALIZATIONS: + return ModelMigrationDecision( + needed=False, reason=f"'{materialized}' models keep no warehouse data" + ) + if after is None or new_model is None: + return ModelMigrationDecision( + needed=True, + reason="keeps the relation's history; the destination could not be checked because " + "the edited project does not compile", + ) + if _location_key(old_model) == _location_key(new_model): + return ModelMigrationDecision(needed=False, reason="relation name is unchanged") + return _blocked_move(after=after, old_model=old_model, new_model=new_model) or ( + ModelMigrationDecision( + needed=True, reason="keeps the relation's history and its old name working" + ) + ) + + +def _blocked_move( + *, after: CompiledProject, old_model: CompiledModel, new_model: CompiledModel +) -> ModelMigrationDecision | None: + old_database: str = (old_model.destination.database or "").lower() + new_database: str = (new_model.destination.database or "").lower() + if old_database and new_database and old_database != new_database: + return ModelMigrationDecision( + needed=False, + blocked=True, + reason=( + f"moves from database {old_model.destination.database} to " + f"{new_model.destination.database}; migrations cannot cross databases" + ), + ) + old_schema: str = (old_model.destination.schema or "").lower() + project_schemas: frozenset[str] = frozenset( + (model.destination.schema or "").lower() for model in after.models + ) + if old_schema in project_schemas: + return None + return ModelMigrationDecision( + needed=False, + blocked=True, + reason=( + f"no model is left in schema {old_model.destination.schema}, so migrate_from " + f"{old_model.name} cannot find the old relation; add a schema-qualified " + "migrate_from for each target" + ), + ) + + +def needs_column_migration(*, model: CompiledModel) -> bool: + """Return whether renaming a column of this model must declare migrate_from.""" + + materialized: str | None = get_config_str(values=model.config.values, key=MATERIALIZED_KEY) + if materialized in HISTORY_MATERIALIZATIONS: + return True + return ( + materialized in MIGRATABLE_MATERIALIZATIONS + and model.config.values.get(MIGRATE_FROM_KEY) is not None + ) + + +def _model(*, project: CompiledProject, name: str) -> CompiledModel | None: + return next((model for model in project.models if model.name == name), None) + + +def _location_key(model: CompiledModel) -> tuple[str, str, str]: + return ( + (model.destination.database or "").lower(), + (model.destination.schema or "").lower(), + model.destination.name.lower(), + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/renames/model_references.py b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_references.py new file mode 100644 index 0000000000..9064c7bd39 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_references.py @@ -0,0 +1,188 @@ +"""Every authored reference to a model by name, and the ones a rename cannot rewrite.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.refactoring._helpers.text.header_edits import model_name_header_edits +from sqlbuild.compiler.refactoring._helpers.text.schema_edits import schema_model_name_edits +from sqlbuild.compiler.refactoring._helpers.text.sql_sites import ( + authored_offset, + model_body, + resource_sites, +) +from sqlbuild.compiler.refactoring._helpers.text.text_edits import ( + identifier_sites, + manual_at, + path_edits, + text_edit, +) +from sqlbuild.compiler.refactoring.constants import ( + EXPECTED_FIXTURE_PREFIX, + FIXTURE_ROLES, + REF_FIXTURE_PREFIX, + REF_KIND, +) +from sqlbuild.compiler.refactoring.models import ( + ManualLocation, + ModelBody, + ProjectSqlFile, + RefactorProject, + ResourceSite, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import EditKind, SqlFileRole + + +def model_reference_edits( + *, files: tuple[ProjectSqlFile, ...], old: str, new: str, dialect: str +) -> tuple[tuple[str, TextEdit], ...]: + """Rewrite `__ref`, fixture CTE names, and header references to a renamed model.""" + + edits: list[tuple[str, TextEdit]] = [] + item: ProjectSqlFile + for item in files: + edits.extend( + path_edits( + path=item.relative_path, + edits=_ref_edits(item=item, old=old, new=new, dialect=dialect), + ) + ) + if item.role in FIXTURE_ROLES: + edits.extend( + path_edits( + path=item.relative_path, edits=_fixture_edits(item=item, old=old, new=new) + ) + ) + if item.role == SqlFileRole.MODEL: + edits.extend( + path_edits( + path=item.relative_path, + edits=model_name_header_edits(contents=item.contents, old=old, new=new), + ) + ) + if item.role == SqlFileRole.SCHEMA: + edits.extend( + path_edits( + path=item.relative_path, + edits=schema_model_name_edits(contents=item.contents, old=old, new=new), + ) + ) + return tuple(edits) + + +def macro_reference_locations( + *, project: RefactorProject, files: tuple[ProjectSqlFile, ...], old: str, dialect: str +) -> tuple[ManualLocation, ...]: + """Locate `__ref` calls to the model that macros produce, which no edit can reach.""" + + compiled: CompiledProject = project.graph.project + contents: dict[str, str] = {item.relative_path: item.contents for item in files} + locations: list[ManualLocation] = [] + model: CompiledModel + for model in compiled.models: + text: str | None = contents.get(model.relative_path.as_posix()) + if text is None or not _depends_on(model=model, name=old): + continue + locations.extend( + _generated_references( + compiled=compiled, project=project, model=model, text=text, old=old, dialect=dialect + ) + ) + return tuple(locations) + + +def _ref_edits(*, item: ProjectSqlFile, old: str, new: str, dialect: str) -> tuple[TextEdit, ...]: + return tuple( + text_edit( + text=item.contents, + start=site.name_start, + end=site.name_end, + replacement=new, + kind=EditKind.REFERENCE, + before=item.contents[site.start : site.end], + after=( + item.contents[site.start : site.name_start] + + new + + item.contents[site.name_end : site.end] + ), + ) + for site in _ref_sites(text=item.contents, name=old, dialect=dialect) + ) + + +def _fixture_edits(*, item: ProjectSqlFile, old: str, new: str) -> tuple[TextEdit, ...]: + fixture_names: dict[str, str] = { + f"{REF_FIXTURE_PREFIX}{old}": f"{REF_FIXTURE_PREFIX}{new}", + f"{EXPECTED_FIXTURE_PREFIX}{old}": f"{EXPECTED_FIXTURE_PREFIX}{new}", + } + return tuple( + text_edit( + text=item.contents, + start=start, + end=end, + replacement=fixture_names[name], + kind=EditKind.FIXTURE, + ) + for start, end, name in identifier_sites(text=item.contents, names=frozenset(fixture_names)) + ) + + +def _generated_references( + *, + compiled: CompiledProject, + project: RefactorProject, + model: CompiledModel, + text: str, + old: str, + dialect: str, +) -> tuple[ManualLocation, ...]: + relative: str = model.relative_path.as_posix() + compiled_sites: tuple[ResourceSite, ...] = _ref_sites( + text=model.query_sql, name=old, dialect=dialect + ) + authored_sites: tuple[ResourceSite, ...] = _ref_sites( + text=model.authored_query_sql, name=old, dialect=dialect + ) + if len(compiled_sites) <= len(authored_sites): + return () + body: ModelBody | None = model_body( + project=compiled, project_dir=project.project_dir, model=model, contents=text + ) + if body is None: + return ( + manual_at( + path=relative, + text=text, + offset=None, + reason=f"reference to {old} produced by a macro", + ), + ) + offsets: tuple[tuple[int, bool], ...] = tuple( + authored_offset(body=body, offset=site.start) for site in compiled_sites + ) + return tuple( + manual_at( + path=relative, + text=text, + offset=offset, + reason=f"reference to {old} produced by a macro; pass the model to the macro as an " + "argument", + ) + for offset, generated in offsets + if generated + ) + + +def _ref_sites(*, text: str, name: str, dialect: str) -> tuple[ResourceSite, ...]: + return tuple( + site + for site in resource_sites(text=text, dialect=dialect) + if site.kind == REF_KIND and site.name == name + ) + + +def _depends_on(*, model: CompiledModel, name: str) -> bool: + return any( + dep.resource_type == CompiledResourceType.MODEL and dep.name == name for dep in model.deps + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/text/__init__.py new file mode 100644 index 0000000000..e0399ee685 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/__init__.py @@ -0,0 +1 @@ +"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/header_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/header_edits.py new file mode 100644 index 0000000000..c7e3fbc272 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/header_edits.py @@ -0,0 +1,449 @@ +"""Token-exact MODEL header edits for renamed models and columns.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.discovery.main._model_header_spans import get_model_header_spans +from sqlbuild.compiler.discovery.models import ModelHeaderSpans +from sqlbuild.compiler.refactoring._helpers.text.sql_sites import embedded_ref_spans +from sqlbuild.compiler.refactoring._helpers.text.text_edits import text_edit, whole_word_offsets +from sqlbuild.compiler.refactoring.constants import ( + COLUMN_VALUED_CONFIG_KEYS, + COLUMNS_KEY, + CURSOR_INPUTS_KEY, + HEADER_CLOSERS, + HEADER_DESCRIPTION_KEY, + HEADER_INDENT, + HEADER_KEY_AND_VALUE_TOKENS, + HEADER_OPEN_PAREN, + HEADER_OPENERS, + HEADER_SEPARATOR, + IDENTIFIER_PATTERN, + NATIVE_HEADER_TOKEN_KINDS, + PARENTHESIZED_EMPTY_TOKENS, + REF_FUNCTION, + RELATIONSHIPS_AUDIT, + RELATIONSHIPS_FIELD_KEY, + RELATIONSHIPS_TO_KEY, +) +from sqlbuild.compiler.refactoring.models import ( + HeaderEntry, + HeaderToken, + RelationshipTokens, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import EditKind, HeaderTokenKind +from sqlbuild.lint.main.quoted_value_end import quoted_value_end + +_VALUE_KINDS: frozenset[HeaderTokenKind] = frozenset({HeaderTokenKind.WORD, HeaderTokenKind.STRING}) + + +def header_tokens(*, contents: str) -> tuple[HeaderToken, ...] | None: + """Tokenize the MODEL header body in file offsets, or None without a header.""" + + spans: ModelHeaderSpans = get_model_header_spans(contents=contents) + if spans.body is None: + return None + return span_tokens(contents=contents, span=spans.body) + + +def span_tokens(*, contents: str, span: tuple[int, int]) -> tuple[HeaderToken, ...]: + """Tokenize one declaration header body in file offsets.""" + + body_start: int = span[0] + body: str = contents[span[0] : span[1]] + tokens: list[HeaderToken] = [] + depth: int = 0 + raw_kind: int + value: str + position: int + for raw_kind, value, position in _native.tokenize_model_header(body): + kind: HeaderTokenKind = NATIVE_HEADER_TOKEN_KINDS[raw_kind] + if kind == HeaderTokenKind.END: + continue + start: int = body_start + position + end: int = _token_end(contents=contents, kind=kind, value=value, start=start) + if kind == HeaderTokenKind.SYMBOL and value in HEADER_CLOSERS: + depth -= 1 + tokens.append(HeaderToken(kind=kind, value=value, start=start, end=end, depth=depth)) + if kind == HeaderTokenKind.SYMBOL and value in HEADER_OPENERS: + depth += 1 + return tuple(tokens) + + +def header_entries(*, tokens: tuple[HeaderToken, ...]) -> tuple[HeaderEntry, ...]: + """Split header tokens into top-level entries keyed by their first word.""" + + return tuple( + HeaderEntry(key=first.value, tokens=(first, *rest)) + for first, rest in _nested_entries(tokens=tokens, depth=0) + if first.kind == HeaderTokenKind.WORD + ) + + +def rename_value_token( + *, contents: str, token: HeaderToken, new_value: str, kind: EditKind +) -> TextEdit: + """Replace one word or string value, keeping string quoting.""" + + replacement: str = ( + new_value + if token.kind == HeaderTokenKind.WORD + else f"{contents[token.start]}{new_value}{contents[token.start]}" + ) + return text_edit( + text=contents, + start=token.start, + end=token.end, + replacement=replacement, + kind=kind, + ) + + +def is_value(*, token: HeaderToken, value: str) -> bool: + """Return whether a word or string token spells a name, ignoring case.""" + + return token.kind in _VALUE_KINDS and token.value.lower() == value.lower() + + +def cursor_input_tokens( + *, tokens: tuple[HeaderToken, ...] +) -> tuple[tuple[HeaderToken, tuple[HeaderToken, ...]], ...]: + """Return (input name token, value tokens) for every cursor_inputs entry.""" + + entry: HeaderEntry + for entry in header_entries(tokens=tokens): + if entry.key != CURSOR_INPUTS_KEY: + continue + return _nested_entries(tokens=entry.tokens[1:], depth=1) + return () + + +def relationship_tokens(*, tokens: tuple[HeaderToken, ...]) -> tuple[RelationshipTokens, ...]: + """Return the target and field value tokens of every relationships audit.""" + + found: list[RelationshipTokens] = [] + index: int + token: HeaderToken + for index, token in enumerate(tokens): + if not ( + token.kind == HeaderTokenKind.WORD + and token.value == RELATIONSHIPS_AUDIT + and index + 1 < len(tokens) + and tokens[index + 1].value == HEADER_OPEN_PAREN + ): + continue + depth: int = tokens[index + 1].depth + 1 + values: dict[str, int] = {} + cursor: int = index + 2 + while cursor < len(tokens) and tokens[cursor].depth >= depth: + candidate: HeaderToken = tokens[cursor] + if ( + candidate.depth == depth + and candidate.kind == HeaderTokenKind.WORD + and candidate.value in {RELATIONSHIPS_TO_KEY, RELATIONSHIPS_FIELD_KEY} + and cursor + 1 < len(tokens) + ): + values[candidate.value] = cursor + 1 + cursor += 1 + found.append( + _relationship( + tokens=tokens, + to_index=values.get(RELATIONSHIPS_TO_KEY), + field_index=values.get(RELATIONSHIPS_FIELD_KEY), + ) + ) + return tuple(found) + + +def column_config_edits( + *, contents: str, tokens: tuple[HeaderToken, ...], old: str, new: str +) -> tuple[TextEdit, ...]: + """Rename a column in column-valued config such as cursor or unique_key.""" + + edits: list[TextEdit] = [] + entry: HeaderEntry + for entry in header_entries(tokens=tokens): + if entry.key not in COLUMN_VALUED_CONFIG_KEYS: + continue + token: HeaderToken + for token in entry.tokens[1:]: + if is_value(token=token, value=old): + edits.append( + rename_value_token( + contents=contents, token=token, new_value=new, kind=EditKind.HEADER + ) + ) + return tuple(edits) + + +def column_entry_edits( + *, contents: str, tokens: tuple[HeaderToken, ...], old: str, new: str, migrate: bool +) -> tuple[TextEdit, ...] | None: + """Rename a `columns` entry and add `migrate_from` if needed; None without an entry.""" + + entry: HeaderEntry + for entry in header_entries(tokens=tokens): + if entry.key != COLUMNS_KEY: + continue + nested: tuple[tuple[HeaderToken, tuple[HeaderToken, ...]], ...] = _nested_entries( + tokens=entry.tokens[1:], depth=1 + ) + name_token: HeaderToken + values: tuple[HeaderToken, ...] + for name_token, values in nested: + if not is_value(token=name_token, value=old): + continue + edits: list[TextEdit] = [ + rename_value_token( + contents=contents, token=name_token, new_value=new, kind=EditKind.HEADER + ) + ] + if migrate: + edits.append( + _column_migration_edit( + contents=contents, name=name_token, values=values, old=old + ) + ) + return tuple(edits) + return None + + +def add_column_entry_edit( + *, contents: str, tokens: tuple[HeaderToken, ...], new: str, old: str +) -> TextEdit | None: + """Declare `new (migrate_from old)` in the columns block, creating it when absent.""" + + declaration: str = f"{new} (migrate_from {old})" + entry: HeaderEntry + for entry in header_entries(tokens=tokens): + if entry.key != COLUMNS_KEY or len(entry.tokens) < HEADER_KEY_AND_VALUE_TOKENS: + continue + opener: HeaderToken = entry.tokens[1] + indentation: str = _line_indent(contents=contents, offset=entry.tokens[0].start) + return text_edit( + text=contents, + start=opener.end, + end=opener.end, + replacement=f"\n{indentation}{HEADER_INDENT}{declaration},", + kind=EditKind.MIGRATION, + before="", + after=declaration, + ) + return insert_header_entry_edit( + contents=contents, entry=f"columns ({declaration})", display=declaration + ) + + +def insert_header_entry_edit(*, contents: str, entry: str, display: str) -> TextEdit | None: + """Insert one top-level entry as the first MODEL header line.""" + + spans: ModelHeaderSpans = get_model_header_spans(contents=contents) + if spans.body is None: + return None + start: int = spans.body[0] + body: str = contents[spans.body[0] : spans.body[1]] + if body.startswith("\n"): + line_end: int = body.find("\n", 1) + first_line: str = body[1:] if line_end < 0 else body[1:line_end] + indentation: str = first_line[: len(first_line) - len(first_line.lstrip())] or HEADER_INDENT + replacement: str = f"\n{indentation}{entry}," + elif body.strip(): + replacement = f"{entry}, " + else: + replacement = f"\n{HEADER_INDENT}{entry},\n" + return text_edit( + text=contents, + start=start, + end=start, + replacement=replacement, + kind=EditKind.MIGRATION, + before="", + after=display, + ) + + +def unhandled_word_offsets( + *, contents: str, tokens: tuple[HeaderToken, ...], word: str, handled: frozenset[int] +) -> tuple[int, ...]: + """Return header offsets that still mention a word outside descriptions.""" + + offsets: list[int] = [] + previous: HeaderToken | None = None + token: HeaderToken + for token in tokens: + described: bool = previous is not None and previous.value == HEADER_DESCRIPTION_KEY + previous = token + if token.kind == HeaderTokenKind.SYMBOL or described or token.start in handled: + continue + text: str = contents[token.start : token.end] + offsets.extend(token.start + offset for offset in whole_word_offsets(text=text, word=word)) + return tuple(offsets) + + +def model_name_header_edits(*, contents: str, old: str, new: str) -> tuple[TextEdit, ...]: + """Rename a model in cursor_inputs keys, bare relationships targets, and quoted SQL.""" + + tokens: tuple[HeaderToken, ...] | None = header_tokens(contents=contents) + if tokens is None: + return () + return model_name_token_edits(contents=contents, tokens=tokens, old=old, new=new) + + +def model_name_token_edits( + *, contents: str, tokens: tuple[HeaderToken, ...], old: str, new: str +) -> tuple[TextEdit, ...]: + """Rename a model in the given header tokens.""" + + names: list[HeaderToken] = [ + name for name, _ in cursor_input_tokens(tokens=tokens) if is_value(token=name, value=old) + ] + names.extend( + relationship.target + for relationship in relationship_tokens(tokens=tokens) + if relationship.target is not None + and not relationship.called + and is_value(token=relationship.target, value=old) + ) + edits: list[TextEdit] = [ + rename_value_token(contents=contents, token=name, new_value=new, kind=EditKind.HEADER) + for name in names + ] + token: HeaderToken + for token in tokens: + if token.kind == HeaderTokenKind.STRING: + edits.extend( + text_edit( + text=contents, + start=token.start + start, + end=token.start + end, + replacement=new, + kind=EditKind.REFERENCE, + ) + for start, end in embedded_ref_spans( + text=contents[token.start : token.end], name=old + ) + ) + return tuple(edits) + + +def consumer_column_header_edits( + *, contents: str, upstream: str, old: str, new: str +) -> tuple[TextEdit, ...]: + """Rename an upstream column in cursor_inputs values and relationships `field` values.""" + + tokens: tuple[HeaderToken, ...] | None = header_tokens(contents=contents) + if tokens is None: + return () + return column_token_edits(contents=contents, tokens=tokens, upstream=upstream, old=old, new=new) + + +def column_token_edits( + *, contents: str, tokens: tuple[HeaderToken, ...], upstream: str, old: str, new: str +) -> tuple[TextEdit, ...]: + """Rename an upstream column in the given header tokens.""" + + values: list[HeaderToken] = [] + name: HeaderToken + inputs: tuple[HeaderToken, ...] + for name, inputs in cursor_input_tokens(tokens=tokens): + if is_value(token=name, value=upstream): + values.extend(value for value in inputs if is_value(token=value, value=old)) + relationship: RelationshipTokens + for relationship in relationship_tokens(tokens=tokens): + if ( + relationship.target is not None + and relationship.field is not None + and is_value(token=relationship.target, value=upstream) + and is_value(token=relationship.field, value=old) + ): + values.append(relationship.field) + return tuple( + rename_value_token(contents=contents, token=value, new_value=new, kind=EditKind.HEADER) + for value in values + ) + + +def quoted_name(name: str) -> str: + """Return a header value spelling for a name.""" + + return name if IDENTIFIER_PATTERN.match(name) else f'"{name}"' + + +def _column_migration_edit( + *, contents: str, name: HeaderToken, values: tuple[HeaderToken, ...], old: str +) -> TextEdit: + declaration: str = f"migrate_from {old}" + if values and values[0].value == HEADER_OPEN_PAREN: + opener: HeaderToken = values[0] + has_metadata: bool = len(values) > PARENTHESIZED_EMPTY_TOKENS + return text_edit( + text=contents, + start=opener.end, + end=opener.end, + replacement=f"{declaration}, " if has_metadata else declaration, + kind=EditKind.MIGRATION, + before="", + after=declaration, + ) + return text_edit( + text=contents, + start=name.end, + end=name.end, + replacement=f" ({declaration})", + kind=EditKind.MIGRATION, + before="", + after=declaration, + ) + + +def _relationship( + *, tokens: tuple[HeaderToken, ...], to_index: int | None, field_index: int | None +) -> RelationshipTokens: + field: HeaderToken | None = tokens[field_index] if field_index is not None else None + if to_index is None: + return RelationshipTokens(target=None, field=field, called=False) + called: bool = ( + tokens[to_index].value == REF_FUNCTION + and to_index + 2 < len(tokens) + and tokens[to_index + 1].value == HEADER_OPEN_PAREN + and tokens[to_index + 2].kind == HeaderTokenKind.STRING + ) + return RelationshipTokens( + target=tokens[to_index + 2] if called else tokens[to_index], field=field, called=called + ) + + +def _nested_entries( + *, tokens: tuple[HeaderToken, ...], depth: int +) -> tuple[tuple[HeaderToken, tuple[HeaderToken, ...]], ...]: + """Split the contents of one bracketed value into (first token, rest) entries.""" + + groups: list[list[HeaderToken]] = [[]] + token: HeaderToken + for token in tokens: + if token.depth < depth: + continue + if token.depth == depth and _is_separator(token): + groups.append([]) + continue + groups[-1].append(token) + return tuple( + (group[0], tuple(group[1:])) for group in groups if group and group[0].kind in _VALUE_KINDS + ) + + +def _is_separator(token: HeaderToken) -> bool: + return token.kind == HeaderTokenKind.SYMBOL and token.value == HEADER_SEPARATOR + + +def _token_end(*, contents: str, kind: HeaderTokenKind, value: str, start: int) -> int: + if kind != HeaderTokenKind.STRING: + return start + len(value) + return quoted_value_end(text=contents, start=start) + + +def _line_indent(*, contents: str, offset: int) -> str: + line_start: int = contents.rfind("\n", 0, offset) + 1 + prefix: str = contents[line_start:offset] + return prefix if not prefix.strip() else HEADER_INDENT diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py new file mode 100644 index 0000000000..98cda55188 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py @@ -0,0 +1,48 @@ +"""Model and column references in reusable SCHEMA declaration headers.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( + column_token_edits, + model_name_token_edits, + span_tokens, +) +from sqlbuild.compiler.refactoring.constants import SCHEMA_STATEMENT_PATTERN +from sqlbuild.compiler.refactoring.models import HeaderToken, TextEdit + + +def schema_model_name_edits(*, contents: str, old: str, new: str) -> tuple[TextEdit, ...]: + """Rename a model in every SCHEMA header of a declaration file.""" + + return _each_schema( + contents=contents, + edit=lambda tokens: model_name_token_edits( + contents=contents, tokens=tokens, old=old, new=new + ), + ) + + +def schema_column_edits( + *, contents: str, upstream: str, old: str, new: str +) -> tuple[TextEdit, ...]: + """Rename relationships `field` values pointing at a renamed column of a model.""" + + return _each_schema( + contents=contents, + edit=lambda tokens: column_token_edits( + contents=contents, tokens=tokens, upstream=upstream, old=old, new=new + ), + ) + + +def _each_schema( + *, contents: str, edit: Callable[[tuple[HeaderToken, ...]], tuple[TextEdit, ...]] +) -> tuple[TextEdit, ...]: + edits: list[TextEdit] = [] + for match in SCHEMA_STATEMENT_PATTERN.finditer(contents): + edits.extend( + edit(span_tokens(contents=contents, span=(match.start("header"), match.end("header")))) + ) + return tuple(edits) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/sql_sites.py b/src/sqlbuild/compiler/refactoring/_helpers/text/sql_sites.py new file mode 100644 index 0000000000..4f209e46a3 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/sql_sites.py @@ -0,0 +1,153 @@ +"""Reference call sites, offset-preserving analysis SQL, and compiled-to-authored offsets.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from sqlbuild.compiler.compile.main.map_expanded_offset import map_expanded_offset +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledProject, + CompiledSqlExpansion, + ExpansionSpan, + MappedOffset, +) +from sqlbuild.compiler.refactoring.constants import ( + EMBEDDED_REF_PATTERN, + GENERIC_PLACEHOLDER_KIND, + PLACEHOLDER_BASE, + PLACEHOLDER_PAD, + RESOURCE_CALL_PATTERN, +) +from sqlbuild.compiler.refactoring.models import AnalysisSql, ModelBody, ResourceSite +from sqlbuild.lint.main.scan_interpolation_sites import scan_interpolation_sites +from sqlbuild.lint.models import InterpolationSite + + +def resource_sites(*, text: str, dialect: str) -> tuple[ResourceSite, ...]: + """Return resource calls outside comments and strings, with their name spans.""" + + sites: list[ResourceSite] = [] + site: InterpolationSite + for site in scan_interpolation_sites(body=text, dialect=dialect): + match: re.Match[str] | None = RESOURCE_CALL_PATTERN.match(site.original_text) + if match is None: + continue + sites.append( + ResourceSite( + kind=match.group("kind"), + name=match.group("name"), + start=site.original_start, + end=site.original_end, + name_start=site.original_start + match.start("name"), + name_end=site.original_start + match.end("name"), + ) + ) + return tuple(sites) + + +def embedded_ref_spans(*, text: str, name: str) -> tuple[tuple[int, int], ...]: + """Return the name spans of `__ref` calls to a model inside a quoted string's raw text.""" + + return tuple( + (match.start("name"), match.end("name")) + for match in EMBEDDED_REF_PATTERN.finditer(text) + if match.group("name") == name + ) + + +def analysis_sql(*, text: str, dialect: str) -> AnalysisSql: + """Replace every interpolation site with an identifier of the same length.""" + + pieces: list[str] = [] + tables: dict[tuple[str, str], set[str]] = {} + copied_to: int = 0 + index: int + site: InterpolationSite + for index, site in enumerate(scan_interpolation_sites(body=text, dialect=dialect)): + match: re.Match[str] | None = RESOURCE_CALL_PATTERN.match(site.original_text) + kind: str = match.group("kind")[0] if match is not None else GENERIC_PLACEHOLDER_KIND + placeholder: str = _placeholder( + kind=kind, index=index, length=site.original_end - site.original_start + ) + if match is not None: + tables.setdefault((match.group("kind"), match.group("name")), set()).add(placeholder) + pieces.extend((text[copied_to : site.original_start], placeholder)) + copied_to = site.original_end + pieces.append(text[copied_to:]) + return AnalysisSql( + sql="".join(pieces), + tables={key: frozenset(value) for key, value in tables.items()}, + ) + + +def model_body( + *, project: CompiledProject, project_dir: Path, model: CompiledModel, contents: str +) -> ModelBody | None: + """Return the mappable compiled body of a model, or None when it cannot be mapped.""" + + body_start: int = contents.rfind(model.authored_query_sql) if model.authored_query_sql else -1 + if body_start < 0: + return None + expansion: CompiledSqlExpansion | None = _expansion( + project=project, project_dir=project_dir, model=model + ) + passes: tuple[tuple[ExpansionSpan, ...], ...] = () + if expansion is not None: + if expansion.expanded_sql != model.query_sql: + return None + passes = expansion.passes + elif model.query_sql != model.authored_query_sql: + return None + return ModelBody( + relative_path=model.relative_path.as_posix(), + contents=contents, + body_start=body_start, + compiled_sql=model.query_sql, + passes=passes, + ) + + +def authored_offset(*, body: ModelBody, offset: int) -> tuple[int, bool]: + """Return the file offset of a compiled offset and whether a macro generated it.""" + + mapped: MappedOffset = map_expanded_offset(offset=offset, passes=body.passes) + return body.body_start + mapped.offset, mapped.generated + + +def authored_span(*, body: ModelBody, start: int, end: int) -> tuple[int, int] | None: + """Return the file span of compiled text written by the author, or None if generated.""" + + mapped_start: int + start_generated: bool + mapped_start, start_generated = authored_offset(body=body, offset=start) + if start_generated: + return None + if end <= start: + return mapped_start, mapped_start + mapped_last: int + last_generated: bool + mapped_last, last_generated = authored_offset(body=body, offset=end - 1) + if last_generated or mapped_last - mapped_start != end - 1 - start: + return None + return mapped_start, mapped_last + 1 + + +def _expansion( + *, project: CompiledProject, project_dir: Path, model: CompiledModel +) -> CompiledSqlExpansion | None: + target: Path = (project_dir / model.relative_path).resolve() + path: Path + expansion: CompiledSqlExpansion + for path, expansion in project.sql_expansions.items(): + if path.resolve() == target: + return expansion + return None + + +def _placeholder(*, kind: str, index: int, length: int) -> str: + base: str = PLACEHOLDER_BASE.format(kind=kind, index=index) + if len(base) > length: + return PLACEHOLDER_PAD * (length - 1) + kind + return base + PLACEHOLDER_PAD * (length - len(base)) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/text_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/text_edits.py new file mode 100644 index 0000000000..a82cd19f21 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/text_edits.py @@ -0,0 +1,193 @@ +"""Offsets, positions, and edit application for authored text.""" + +from __future__ import annotations + +import re +from functools import cache + +from sqlbuild.compiler.refactoring.constants import IDENTIFIER_CHARACTERS, NON_CODE_PATTERN +from sqlbuild.compiler.refactoring.exceptions import RefactorEditError +from sqlbuild.compiler.refactoring.models import ( + FileChange, + ManualLocation, + MigrationDeclaration, + RefactorParts, + RefactorPlan, + RefactorRequest, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import EditKind + + +def line_column(*, text: str, offset: int) -> tuple[int, int]: + """Return the 1-based line and column of an offset.""" + + line_start: int = text.rfind("\n", 0, offset) + 1 + return text.count("\n", 0, offset) + 1, offset - line_start + 1 + + +def text_edit( + *, + text: str, + start: int, + end: int, + replacement: str, + kind: EditKind, + before: str | None = None, + after: str | None = None, +) -> TextEdit: + """Build one edit with its position and display text.""" + + line: int + column: int + line, column = line_column(text=text, offset=start) + return TextEdit( + start=start, + end=end, + replacement=replacement, + kind=kind, + line=line, + column=column, + before=text[start:end] if before is None else before, + after=replacement.strip() if after is None else after, + ) + + +def manual_at(*, path: str, text: str, offset: int | None, reason: str) -> ManualLocation: + """Build a manual location at an optional offset.""" + + if offset is None: + return ManualLocation(path=path, line=None, column=None, reason=reason) + line: int + column: int + line, column = line_column(text=text, offset=offset) + return ManualLocation(path=path, line=line, column=column, reason=reason) + + +def apply_text_edits(*, text: str, edits: tuple[TextEdit, ...]) -> str: + """Apply non-overlapping edits back to front; identical duplicates apply once.""" + + unique: list[TextEdit] = sorted( + {(edit.start, edit.end, edit.replacement): edit for edit in edits}.values(), + key=lambda edit: (edit.start, edit.end), + reverse=True, + ) + result: str = text + previous_start: int = len(text) + 1 + edit: TextEdit + for edit in unique: + if edit.end > previous_start: + raise RefactorEditError(f"overlapping refactoring edits at offset {edit.start}") + result = result[: edit.start] + edit.replacement + result[edit.end :] + previous_start = edit.start + return result + + +def merge_parts(*, parts: tuple[RefactorParts, ...]) -> RefactorParts: + """Concatenate planning parts in order.""" + + edits: list[tuple[str, TextEdit]] = [] + manual: list[ManualLocation] = [] + blocking: list[ManualLocation] = [] + migrations: list[MigrationDeclaration] = [] + cascaded: list[str] = [] + part: RefactorParts + for part in parts: + edits.extend(part.edits) + manual.extend(part.manual) + blocking.extend(part.blocking) + migrations.extend(part.migrations) + cascaded.extend(part.cascaded) + return RefactorParts( + edits=tuple(edits), + manual=tuple(manual), + blocking=tuple(blocking), + migrations=tuple(migrations), + cascaded=tuple(cascaded), + ) + + +def path_edits(*, path: str, edits: tuple[TextEdit, ...]) -> tuple[tuple[str, TextEdit], ...]: + """Attach a file path to each edit.""" + + return tuple((path, edit) for edit in edits) + + +def file_changes( + *, edits: tuple[tuple[str, TextEdit], ...], moves: dict[str, str] +) -> tuple[FileChange, ...]: + """Group edits by file, move files, and order the result by final path.""" + + grouped: dict[str, list[TextEdit]] = {path: [] for path in moves} + path: str + edit: TextEdit + for path, edit in edits: + grouped.setdefault(path, []).append(edit) + return tuple( + sorted( + ( + FileChange(path=moves.get(path, path), original_path=path, edits=tuple(items)) + for path, items in grouped.items() + ), + key=lambda change: change.path, + ) + ) + + +def build_plan( + *, + request: RefactorRequest, + parts: RefactorParts, + moves: dict[str, str], + renamed_columns: tuple[tuple[str, str, str], ...] = (), + help: str | None = None, +) -> RefactorPlan: + """Assemble a plan from merged parts; help applies only when manual locations exist.""" + + return RefactorPlan( + request=request, + changes=file_changes(edits=parts.edits, moves=moves), + manual=parts.manual, + blocking=parts.blocking, + migrations=parts.migrations, + renamed_columns=renamed_columns, + help=help if parts.manual else None, + ) + + +def identifier_sites(*, text: str, names: frozenset[str]) -> tuple[tuple[int, int, str], ...]: + """Return unquoted identifier tokens matching names, outside comments and strings.""" + + if not names: + return () + lowered: dict[str, str] = {name.lower(): name for name in names} + sites: list[tuple[int, int, str]] = [] + match: re.Match[str] + for match in _identifier_pattern(frozenset(lowered)).finditer(text): + token: str | None = match.group("identifier") + if token is not None: + sites.append( + (match.start("identifier"), match.end("identifier"), lowered[token.lower()]) + ) + return tuple(sites) + + +def whole_word_offsets(*, text: str, word: str) -> tuple[int, ...]: + """Return the offsets of a case-insensitive whole identifier word anywhere in text.""" + + pattern: re.Pattern[str] = re.compile( + rf"(? re.Pattern[str]: + ordered: list[str] = sorted(names, key=lambda name: (-len(name), name)) + alternatives: str = "|".join(re.escape(name) for name in ordered) + return re.compile( + NON_CODE_PATTERN + + rf"|(?{alternatives})" + + rf"(?![{IDENTIFIER_CHARACTERS}])", + re.I, + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/yaml_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/yaml_edits.py new file mode 100644 index 0000000000..aec4de9fa9 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/yaml_edits.py @@ -0,0 +1,171 @@ +"""Model and column references in source and seed YAML declarations.""" + +from __future__ import annotations + +import yaml +from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode + +from sqlbuild.compiler.refactoring._helpers.text.sql_sites import embedded_ref_spans +from sqlbuild.compiler.refactoring._helpers.text.text_edits import path_edits, text_edit +from sqlbuild.compiler.refactoring.constants import ( + RELATIONSHIPS_AUDIT, + RELATIONSHIPS_FIELD_KEY, + RELATIONSHIPS_TO_KEY, +) +from sqlbuild.compiler.refactoring.models import ProjectSqlFile, TextEdit, YamlRelationship +from sqlbuild.compiler.refactoring.types import EditKind + + +def yaml_model_edits( + *, files: tuple[ProjectSqlFile, ...], old: str, new: str +) -> tuple[tuple[str, TextEdit], ...]: + """Rename `__ref` calls in YAML strings and bare relationships targets naming a model.""" + + edits: list[tuple[str, TextEdit]] = [] + item: ProjectSqlFile + for item in files: + root: Node | None = _compose(item.contents) + if root is None: + continue + found: list[TextEdit] = [] + scalar: ScalarNode + for scalar in _scalars(root): + found.extend( + text_edit( + text=item.contents, + start=_start(scalar) + start, + end=_start(scalar) + end, + replacement=new, + kind=EditKind.REFERENCE, + ) + for start, end in embedded_ref_spans( + text=_raw(contents=item.contents, scalar=scalar), name=old + ) + ) + found.extend( + _rename_scalar( + contents=item.contents, + scalar=relationship.target, + new=new, + kind=EditKind.REFERENCE, + ) + for relationship in _relationships(root) + if relationship.target is not None and relationship.target.value == old + ) + edits.extend(path_edits(path=item.relative_path, edits=tuple(found))) + return tuple(edits) + + +def yaml_column_edits( + *, files: tuple[ProjectSqlFile, ...], model: str, old: str, new: str +) -> tuple[tuple[str, TextEdit], ...]: + """Rename relationships `field` values that point at a renamed column of a model.""" + + edits: list[tuple[str, TextEdit]] = [] + item: ProjectSqlFile + for item in files: + root: Node | None = _compose(item.contents) + if root is None: + continue + edits.extend( + path_edits( + path=item.relative_path, + edits=tuple( + _rename_scalar( + contents=item.contents, + scalar=relationship.field, + new=new, + kind=EditKind.COLUMN, + ) + for relationship in _relationships(root) + if relationship.field is not None + and relationship.field.value.lower() == old.lower() + and _targets(contents=item.contents, relationship=relationship, model=model) + ), + ) + ) + return tuple(edits) + + +def _compose(contents: str) -> Node | None: + try: + return yaml.compose(contents, Loader=yaml.SafeLoader) + except yaml.YAMLError: + return None + + +def _scalars(node: Node) -> tuple[ScalarNode, ...]: + if isinstance(node, ScalarNode): + return (node,) + children: list[Node] = [] + if isinstance(node, SequenceNode): + children.extend(node.value) + if isinstance(node, MappingNode): + children.extend(value for _, value in node.value) + found: list[ScalarNode] = [] + child: Node + for child in children: + found.extend(_scalars(child)) + return tuple(found) + + +def _relationships(node: Node) -> tuple[YamlRelationship, ...]: + found: list[YamlRelationship] = [] + if isinstance(node, MappingNode): + key: Node + value: Node + for key, value in node.value: + if ( + isinstance(key, ScalarNode) + and key.value == RELATIONSHIPS_AUDIT + and isinstance(value, MappingNode) + ): + found.append(_relationship(value)) + found.extend(_relationships(value)) + if isinstance(node, SequenceNode): + child: Node + for child in node.value: + found.extend(_relationships(child)) + return tuple(found) + + +def _relationship(node: MappingNode) -> YamlRelationship: + values: dict[str, ScalarNode] = { + key.value: value + for key, value in node.value + if isinstance(key, ScalarNode) and isinstance(value, ScalarNode) + } + return YamlRelationship( + target=values.get(RELATIONSHIPS_TO_KEY), field=values.get(RELATIONSHIPS_FIELD_KEY) + ) + + +def _targets(*, contents: str, relationship: YamlRelationship, model: str) -> bool: + target: ScalarNode | None = relationship.target + if target is None: + return False + return target.value == model or bool( + embedded_ref_spans(text=_raw(contents=contents, scalar=target), name=model) + ) + + +def _rename_scalar(*, contents: str, scalar: ScalarNode, new: str, kind: EditKind) -> TextEdit: + raw: str = _raw(contents=contents, scalar=scalar) + offset: int = max(raw.lower().find(scalar.value.lower()), 0) + start: int = _start(scalar) + offset + return text_edit( + text=contents, + start=start, + end=start + len(scalar.value), + replacement=new, + kind=kind, + ) + + +def _raw(*, contents: str, scalar: ScalarNode) -> str: + end: int = scalar.end_mark.index if scalar.end_mark is not None else _start(scalar) + return contents[_start(scalar) : end] + + +def _start(scalar: ScalarNode) -> int: + return scalar.start_mark.index if scalar.start_mark is not None else 0 diff --git a/src/sqlbuild/compiler/refactoring/constants.py b/src/sqlbuild/compiler/refactoring/constants.py new file mode 100644 index 0000000000..7cee4805a0 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/constants.py @@ -0,0 +1,112 @@ +"""Constants for project refactorings.""" + +from __future__ import annotations + +import re + +from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN +from sqlbuild.compiler.refactoring.types import ( + HeaderTokenKind, + RefactorOperation, + SqlFileRole, +) + +MODEL_KIND_PREFIX: str = "model:" +COLUMN_KIND_PREFIX: str = "column:" +SQL_SUFFIX: str = ".sql" +PATH_SEPARATOR: str = "/" +IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +REF_FUNCTION: str = "__ref" +REF_CALL_PATTERN: re.Pattern[str] = re.compile( + r"^__ref\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" +) +RESOURCE_CALL_PATTERN: re.Pattern[str] = re.compile( + r"^__(?Pref|source|seed)\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" +) +EMBEDDED_REF_PATTERN: re.Pattern[str] = re.compile( + r"""__ref\s*\(\s*\\?['"](?P[^'"\\]+)\\?['"]\s*\)""" +) +REF_FIXTURE_PREFIX: str = "__ref__" +EXPECTED_FIXTURE_PREFIX: str = "__expected__" +ASSERT_FIXTURE_PREFIX: str = "__assert__" +NON_CODE_PATTERN: str = ( + r"--[^\n]*(?:\n|\Z)" + r"|/\*[\s\S]*?(?:\*/|\Z)" + r"|'(?:''|[^'])*(?:'|\Z)" + r'|"(?:""|[^"])*(?:"|\Z)' +) +IDENTIFIER_CHARACTERS: str = r"A-Za-z0-9_$" +PLACEHOLDER_PAD: str = "_" +GENERIC_PLACEHOLDER_KIND: str = "x" +ROOT_SCOPE: str = "root" +CTE_SCOPE_PREFIX: str = "cte:" +MIGRATE_FROM_KEY: str = "migrate_from" +MATERIALIZED_KEY: str = "materialized" +CURSOR_INPUTS_KEY: str = "cursor_inputs" +COLUMNS_KEY: str = "columns" +RELATIONSHIPS_AUDIT: str = "relationships" +RELATIONSHIPS_TO_KEY: str = "to" +RELATIONSHIPS_FIELD_KEY: str = "field" +COLUMN_VALUED_CONFIG_KEYS: frozenset[str] = frozenset( + { + "cursor", + "unique_key", + "updated_at", + "check_columns", + "merge_exclude_columns", + "partition_by", + "cluster_by", + "observed_at", + "row_diff_exclude_columns", + } +) +HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset({"incremental", "snapshot"}) +MIGRATABLE_MATERIALIZATIONS: frozenset[str] = frozenset( + {"table", "view", "incremental", "snapshot"} +) +COPIED_PROJECT_SUFFIXES: frozenset[str] = frozenset( + {".sql", ".py", ".yml", ".yaml", ".toml", ".csv", ".json"} +) +IGNORED_PROJECT_DIRECTORIES: frozenset[str] = frozenset( + {"target", "logs", "venv", "node_modules", "__pycache__"} +) +SQL_LIKE_PATTERN: re.Pattern[str] = re.compile(r"\b(select|from|join|insert|update)\b", re.I) +OPERATION_TITLES: dict[RefactorOperation, str] = { + RefactorOperation.RENAME_MODEL: "Rename model", + RefactorOperation.MOVE_MODEL: "Move model", + RefactorOperation.RENAME_COLUMN: "Rename column", +} +PLACEHOLDER_BASE: str = "_q{kind}{index}" +QUOTE_CHARACTERS: dict[str, str] = {'"': '"', "`": "`", "[": "]"} +COLUMN_TARGET_SEPARATOR: str = "." +HEADER_OPENERS: frozenset[str] = frozenset({"(", "[", "{"}) +HEADER_CLOSERS: frozenset[str] = frozenset({")", "]", "}"}) +HEADER_SEPARATOR: str = "," +HEADER_OPEN_PAREN: str = "(" +HEADER_DESCRIPTION_KEY: str = "description" +HEADER_INDENT: str = " " +PARENTHESIZED_EMPTY_TOKENS: int = 2 +REF_KIND: str = "ref" +FIXTURE_ROLES: frozenset[SqlFileRole] = frozenset({SqlFileRole.TEST, SqlFileRole.SCENARIO}) +HIDDEN_PREFIX: str = "." +TEXT_ENCODING: str = "utf-8" +GENERIC_DIALECT: str = "generic" +NATIVE_HEADER_TOKEN_KINDS: dict[int, HeaderTokenKind] = { + 0: HeaderTokenKind.END, + 1: HeaderTokenKind.WORD, + 2: HeaderTokenKind.STRING, + 3: HeaderTokenKind.SYMBOL, +} +HEADER_KEY_AND_VALUE_TOKENS: int = 2 +PYCACHE_DIRECTORY: str = "__pycache__" +SOURCE_KIND: str = "source" +SEED_KIND: str = "seed" +UNKNOWN_COLUMN_TYPE: str = "UNKNOWN" +MODEL_MANUAL_HELP: str = ( + "Pass the model into the macro as an argument, or edit the listed locations, then run the " + "command again." +) +COLUMN_MANUAL_HELP: str = "Edit the listed locations, then run the command again." +SCHEMA_STATEMENT_PATTERN: re.Pattern[str] = re.compile( + r"\bSCHEMA\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;" +) diff --git a/src/sqlbuild/compiler/refactoring/exceptions.py b/src/sqlbuild/compiler/refactoring/exceptions.py new file mode 100644 index 0000000000..4226649f3f --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/exceptions.py @@ -0,0 +1,27 @@ +"""Expected refactoring errors.""" + +from __future__ import annotations + + +class RefactorInputError(ValueError): + """A rename or move request that cannot be planned at all.""" + + code: str = "C950" + + def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: + super().__init__(message) + self.message = message + self.code = code if code is not None else self.code + self.help = help + + +class RefactorEditError(RefactorInputError): + """Two planned edits overlap, which a correct plan never produces.""" + + code: str = "C957" + + +class RefactorWriteError(RefactorInputError): + """Project files changed between planning and writing.""" + + code: str = "C958" diff --git a/src/sqlbuild/compiler/refactoring/main/__init__.py b/src/sqlbuild/compiler/refactoring/main/__init__.py new file mode 100644 index 0000000000..fe1ccd557d --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/__init__.py @@ -0,0 +1 @@ +"""Refactoring entry modules.""" diff --git a/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py b/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py new file mode 100644 index 0000000000..8b8ec24ce7 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py @@ -0,0 +1,16 @@ +"""Write a verified refactoring plan into the project.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.refactoring._helpers.project.workspace import commit_changes +from sqlbuild.compiler.refactoring.models import RefactorPlan + + +def commit_refactor_plan( + *, project_dir: Path, originals: dict[str, str], plan: RefactorPlan +) -> tuple[Path, ...]: + """Write every change, restoring all files if any write fails, and return written paths.""" + + return commit_changes(project_dir=project_dir, originals=originals, changes=plan.changes) diff --git a/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py b/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py new file mode 100644 index 0000000000..40c92ac8ba --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py @@ -0,0 +1,31 @@ +"""Plan a column rename from compiler facts, one step or cascading through pass-throughs.""" + +from __future__ import annotations + +from sqlbuild.compiler.refactoring._helpers.columns.column_planning import ( + column_rename_context, + column_rename_parts, +) +from sqlbuild.compiler.refactoring._helpers.text.text_edits import build_plan +from sqlbuild.compiler.refactoring.constants import COLUMN_MANUAL_HELP +from sqlbuild.compiler.refactoring.models import ( + ColumnRenameContext, + RefactorParts, + RefactorPlan, + RefactorProject, + RefactorRequest, +) + + +def plan_column_rename(*, project: RefactorProject, request: RefactorRequest) -> RefactorPlan: + """Plan every edit that renames one output column and its downstream references.""" + + context: ColumnRenameContext = column_rename_context(project=project, request=request) + parts: RefactorParts = column_rename_parts(context=context) + return build_plan( + request=request, + parts=parts, + moves={}, + renamed_columns=tuple((model, context.old, context.new) for model in parts.cascaded), + help=COLUMN_MANUAL_HELP, + ) diff --git a/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py b/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py new file mode 100644 index 0000000000..9770d28eb5 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py @@ -0,0 +1,34 @@ +"""Plan a model rename or move from compiler facts.""" + +from __future__ import annotations + +from sqlbuild.compiler.refactoring._helpers.renames.model_planning import model_parts, model_target +from sqlbuild.compiler.refactoring._helpers.text.text_edits import build_plan +from sqlbuild.compiler.refactoring.constants import MODEL_MANUAL_HELP +from sqlbuild.compiler.refactoring.models import ( + ModelTarget, + RefactorParts, + RefactorPlan, + RefactorProject, + RefactorRequest, +) + + +def plan_model_refactor(*, project: RefactorProject, request: RefactorRequest) -> RefactorPlan: + """Plan every edit that renames or moves one model, without touching files.""" + + target: ModelTarget = model_target(project=project, request=request) + parts: RefactorParts = model_parts(project=project, target=target) + return build_plan( + request=target.request, + parts=parts, + moves={ + **dict(parts.moves), + **( + {target.source_path: target.destination} + if target.destination != target.source_path + else {} + ), + }, + help=MODEL_MANUAL_HELP, + ) diff --git a/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py b/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py new file mode 100644 index 0000000000..c0aa7be537 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py @@ -0,0 +1,24 @@ +"""Stage a refactoring plan in a scratch copy of the project.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.refactoring._helpers.project.workspace import ( + copy_project_inputs, + read_originals, + write_staged_changes, +) +from sqlbuild.compiler.refactoring.models import RefactorPlan + + +def stage_refactor_plan( + *, project_dir: Path, staging_dir: Path, plan: RefactorPlan, copy_inputs: bool = True +) -> dict[str, str]: + """Copy the project inputs once, apply the plan there, and return the original texts.""" + + if copy_inputs: + _ = copy_project_inputs(project_dir=project_dir, staging_dir=staging_dir) + originals: dict[str, str] = read_originals(project_dir=project_dir, changes=plan.changes) + _ = write_staged_changes(staging_dir=staging_dir, originals=originals, changes=plan.changes) + return originals diff --git a/src/sqlbuild/compiler/refactoring/main/with_model_migration.py b/src/sqlbuild/compiler/refactoring/main/with_model_migration.py new file mode 100644 index 0000000000..2b9feb50f5 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/with_model_migration.py @@ -0,0 +1,80 @@ +"""Add model migrate_from to a staged rename whenever its relation moves.""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.refactoring._helpers.renames.model_planning import decide_model_migration +from sqlbuild.compiler.refactoring._helpers.text.header_edits import insert_header_entry_edit +from sqlbuild.compiler.refactoring.constants import MIGRATE_FROM_KEY +from sqlbuild.compiler.refactoring.models import ( + FileChange, + ManualLocation, + MigrationDeclaration, + ModelMigrationDecision, + RefactorPlan, + TextEdit, +) + + +def with_model_migration( + *, + plan: RefactorPlan, + before: CompiledProject, + after: CompiledProject | None, + originals: dict[str, str], +) -> RefactorPlan: + """Return the plan with `migrate_from ` on the model whenever its relation moves.""" + + old: str = plan.request.model_name + new: str = plan.request.new_name + decision: ModelMigrationDecision = decide_model_migration( + before=before, after=after, old=old, new=new + ) + change: FileChange | None = next( + (item for item in plan.changes if item.path == plan.request.destination), None + ) + if decision.blocked: + return replace( + plan, + blocking=( + *plan.blocking, + ManualLocation( + path=plan.request.destination or "", + line=None, + column=None, + reason=decision.reason, + ), + ), + ) + if not decision.needed or change is None: + return plan + declaration: str = f"{MIGRATE_FROM_KEY} {old}" + edit: TextEdit | None = insert_header_entry_edit( + contents=originals[change.original_path], entry=declaration, display=declaration + ) + if edit is None: + return replace( + plan, + blocking=( + *plan.blocking, + ManualLocation( + path=change.path, + line=1, + column=1, + reason=f"add {declaration} to the MODEL header to keep history", + ), + ), + ) + return replace( + plan, + changes=tuple( + replace(item, edits=(*item.edits, edit)) if item is change else item + for item in plan.changes + ), + migrations=( + *plan.migrations, + MigrationDeclaration(model_name=new, declaration=declaration, reason=decision.reason), + ), + ) diff --git a/src/sqlbuild/compiler/refactoring/models.py b/src/sqlbuild/compiler/refactoring/models.py new file mode 100644 index 0000000000..02c1c82f27 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/models.py @@ -0,0 +1,317 @@ +"""Immutable models for planned and applied project refactorings.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from yaml.nodes import ScalarNode + +from sqlbuild.compiler.compile.models import CompiledModel, ExpansionSpan +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.compiler.refactoring.types import ( + EditKind, + HeaderTokenKind, + OffsetLocator, + RefactorOperation, + ResourceColumns, + SpanMapper, + SqlFileRole, +) + + +@dataclass(frozen=True) +class RefactorProject: + """Compiler facts of one project, as the refactoring planner reads them.""" + + project_dir: Path + graph: ProjectGraph + discovered: DiscoveredProjectInputs + + +@dataclass(frozen=True) +class RefactorRequest: + """One requested rename or move.""" + + operation: RefactorOperation + model_name: str + new_name: str + column_name: str | None = None + destination: str | None = None + cascade: bool = False + + +@dataclass(frozen=True) +class ModelTarget: + """The model a request names, and the request with its name and destination resolved.""" + + model: CompiledModel + request: RefactorRequest + source_path: str + + @property + def destination(self) -> str: + return self.request.destination or self.source_path + + +@dataclass(frozen=True) +class TextEdit: + """One replacement in an authored file, in offsets of the file before any edit.""" + + start: int + end: int + replacement: str + kind: EditKind + line: int + column: int + before: str + after: str + + +@dataclass(frozen=True) +class FileChange: + """Every edit to one file, and where the file lives afterwards.""" + + path: str + original_path: str + edits: tuple[TextEdit, ...] = field(default_factory=tuple) + + @property + def moved(self) -> bool: + return self.path != self.original_path + + +@dataclass(frozen=True) +class ManualLocation: + """A location a refactoring cannot rewrite safely.""" + + path: str + line: int | None + column: int | None + reason: str + + +@dataclass(frozen=True) +class MigrationDeclaration: + """A migrate_from declaration the refactoring adds to keep history.""" + + model_name: str + declaration: str + reason: str + + +@dataclass(frozen=True) +class RefactorPlan: + """Every edit, manual location, and migration of one refactoring.""" + + request: RefactorRequest + changes: tuple[FileChange, ...] = field(default_factory=tuple) + manual: tuple[ManualLocation, ...] = field(default_factory=tuple) + blocking: tuple[ManualLocation, ...] = field(default_factory=tuple) + migrations: tuple[MigrationDeclaration, ...] = field(default_factory=tuple) + renamed_columns: tuple[tuple[str, str, str], ...] = field(default_factory=tuple) + help: str | None = None + + +@dataclass(frozen=True) +class RefactorParts: + """Edits and findings from one planning step, merged into a plan at the end.""" + + edits: tuple[tuple[str, TextEdit], ...] = field(default_factory=tuple) + manual: tuple[ManualLocation, ...] = field(default_factory=tuple) + blocking: tuple[ManualLocation, ...] = field(default_factory=tuple) + migrations: tuple[MigrationDeclaration, ...] = field(default_factory=tuple) + cascaded: tuple[str, ...] = field(default_factory=tuple) + moves: tuple[tuple[str, str], ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class ProjectSqlFile: + """One authored SQL file and the role discovery gave it.""" + + relative_path: str + contents: str + role: SqlFileRole + + +@dataclass(frozen=True) +class AuthoredBody: + """One SQL body inside an authored non-model file.""" + + path: str + role: SqlFileRole + contents: str + start: int + text: str + + +@dataclass(frozen=True) +class ResourceSite: + """One `__ref`, `__source`, or `__seed` call written in authored SQL.""" + + kind: str + name: str + start: int + end: int + name_start: int + name_end: int + + +@dataclass(frozen=True) +class AnalysisSql: + """SQL whose interpolation sites were replaced by same-length identifiers.""" + + sql: str + tables: dict[tuple[str, str], frozenset[str]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ModelBody: + """A model's compiled query and how it maps onto the authored file.""" + + relative_path: str + contents: str + body_start: int + compiled_sql: str + passes: tuple[tuple[ExpansionSpan, ...], ...] + + +@dataclass(frozen=True) +class HeaderToken: + """One MODEL header token in file offsets, with its nesting depth.""" + + kind: HeaderTokenKind + value: str + start: int + end: int + depth: int + + +@dataclass(frozen=True) +class RelationshipTokens: + """The target and field value tokens of one relationships audit in a MODEL header.""" + + target: HeaderToken | None + field: HeaderToken | None + called: bool + + +@dataclass(frozen=True) +class YamlRelationship: + """The `to` and `field` scalars of one relationships audit in a YAML declaration.""" + + target: ScalarNode | None + field: ScalarNode | None + + +@dataclass(frozen=True) +class HeaderEntry: + """One top-level `key value` entry of a MODEL header.""" + + key: str + tokens: tuple[HeaderToken, ...] + + +@dataclass(frozen=True) +class ColumnReference: + """One column expression that resolves to the renamed column.""" + + start: int + end: int + name_start: int + name_end: int + scope: str + projected: bool + alias: str | None + alias_start: int | None + alias_end: int | None + + +@dataclass(frozen=True) +class ColumnSite: + """A scoped construct at an optional span.""" + + scope: str + start: int | None + end: int | None + + +@dataclass(frozen=True) +class OutputColumn: + """One projection that names the column in a requested output scope.""" + + scope: str + start: int | None + end: int | None + alias_start: int | None + alias_end: int | None + + +@dataclass(frozen=True) +class ColumnFacts: + """Every rename-relevant fact about one column in one SQL body.""" + + parsed: bool + references: tuple[ColumnReference, ...] = field(default_factory=tuple) + stars: tuple[ColumnSite, ...] = field(default_factory=tuple) + joins: tuple[ColumnSite, ...] = field(default_factory=tuple) + unresolved: tuple[ColumnSite, ...] = field(default_factory=tuple) + outputs: tuple[OutputColumn, ...] = field(default_factory=tuple) + star_outputs: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class ColumnQuery: + """Which column of which relations one analysis resolves.""" + + column: str + target_tables: frozenset[str] + target_ctes: frozenset[str] = frozenset() + output_scopes: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class BodyContext: + """Where one analysed SQL body lives, and how its offsets map onto the file.""" + + path: str + contents: str + map_span: SpanMapper + locate: OffsetLocator + fallback_offset: int + + +@dataclass(frozen=True) +class BodyEdits: + """Edits, manual locations, and root pass-throughs found in one SQL body.""" + + edits: tuple[TextEdit, ...] = field(default_factory=tuple) + manual: tuple[ManualLocation, ...] = field(default_factory=tuple) + passes_through: bool = False + + +@dataclass(frozen=True) +class ColumnRenameContext: + """Everything a column rename reads while it walks downstream models.""" + + project: RefactorProject + owner: CompiledModel + dialect: str + columns: ResourceColumns + contents: dict[str, str] + bodies: tuple[AuthoredBody, ...] + yaml_files: tuple[ProjectSqlFile, ...] + schema_files: tuple[ProjectSqlFile, ...] + old: str + new: str + cascade: bool + + +@dataclass(frozen=True) +class ModelMigrationDecision: + """Whether a model needs migrate_from, and why, or why none can be written.""" + + needed: bool + reason: str + blocked: bool = False diff --git a/src/sqlbuild/compiler/refactoring/types.py b/src/sqlbuild/compiler/refactoring/types.py new file mode 100644 index 0000000000..9377e1f1ed --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/types.py @@ -0,0 +1,67 @@ +"""Enums and type aliases for project refactorings.""" + +from __future__ import annotations + +from collections.abc import Callable +from enum import StrEnum +from typing import Protocol + + +class RefactorOperation(StrEnum): + """One supported refactoring.""" + + RENAME_MODEL = "rename_model" + MOVE_MODEL = "move_model" + RENAME_COLUMN = "rename_column" + + +class RefactorStatus(StrEnum): + """Terminal state of one refactoring command.""" + + APPLIED = "applied" + DRY_RUN = "dry_run" + REFUSED = "refused" + COMPILE_FAILED = "compile_failed" + + +class EditKind(StrEnum): + """What one text edit changes, for output grouping.""" + + REFERENCE = "reference" + FIXTURE = "fixture" + HEADER = "header" + COLUMN = "column" + MIGRATION = "migration" + + +class SqlFileRole(StrEnum): + """What an authored SQL file declares.""" + + MODEL = "model" + TEST = "test" + SCENARIO = "scenario" + AUDIT = "audit" + HOOK = "hook" + FUNCTION = "function" + SCHEMA = "schema" + YAML = "yaml" + + +class HeaderTokenKind(StrEnum): + """MODEL header token kinds.""" + + END = "end" + WORD = "word" + STRING = "string" + SYMBOL = "symbol" + + +type ResourceColumns = dict[tuple[str, str], tuple[str, ...]] +type SpanMapper = Callable[[int, int], tuple[int, int] | None] +type OffsetLocator = Callable[[int], int] + + +class NativeColumnReferences(Protocol): + """Native column-reference analysis boundary.""" + + def analyze_column_references_json(self, request_json: str) -> str: ... diff --git a/src/sqlbuild/compiler/scopes/_helpers/placement.py b/src/sqlbuild/compiler/scopes/_helpers/placement.py index 749865dd88..d2d1296274 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/placement.py +++ b/src/sqlbuild/compiler/scopes/_helpers/placement.py @@ -172,6 +172,102 @@ def resolve_required_placement( ) +def declaration_directory(*, kind: DeclarationKind, scope: ScopeKind, path: str | None) -> str: + """Return the project-relative folder where a declaration with this placement lives.""" + + role_parts: tuple[str, ...] = DECLARATION_ROLE_PARTS[kind] + if scope is ScopeKind.GLOBAL or path is None: + return "/".join(role_parts) + prefix: str = "_" if scope is ScopeKind.LOCAL else "" + role: str = "/".join((f"{prefix}{role_parts[0]}", *role_parts[1:])) + if path in _AUTHORED_ROOT_PATHS: + return f"{path}/{role}" + return f"{path}/{DECLARATION_GROUP_DIRECTORY}/{role}" + + +def relocate_for_resource_move( + *, index: ScopeIndex, resource: ResourceIdentity, destination: str +) -> tuple[DeclarationRecord, ...] | None: + """Return declarations whose placement changes when a resource moves, or None if unsettled.""" + + if not index.completeness.runtime_usage: + return None + misplaced: frozenset[DeclarationIdentity] = frozenset( + item.declaration + for item in build_placement_validated_index(index=index).diagnostics + if item.code in _PLACEMENT_CODES and item.declaration is not None + ) + current: ScopeIndex = replace( + index, + resources=tuple( + replace(item, path=destination) if item.identity == resource else item + for item in index.resources + ), + ) + usages_by_declaration: _UsagesByDeclaration = _usages_by_declaration(index=current) + for _ in range(len(index.declarations) + 1): + anchor_sets: _AnchorSets = _anchor_sets( + index=current, usages_by_declaration=usages_by_declaration + ) + relocated: dict[DeclarationIdentity, DeclarationRecord] = {} + record: DeclarationRecord + for record in current.declarations: + if record.scope is ScopeKind.PRIVATE or record.identity in misplaced: + continue + moved: DeclarationRecord | None = _relocated_record( + record=record, + usages_by_declaration=usages_by_declaration, + anchor_sets=anchor_sets, + ) + if moved is not None: + relocated[record.identity] = moved + if not relocated: + original: dict[DeclarationIdentity, DeclarationRecord] = { + item.identity: item for item in index.declarations + } + return tuple( + item for item in current.declarations if item.path != original[item.identity].path + ) + current = replace( + current, + declarations=tuple(relocated.get(item.identity, item) for item in current.declarations), + ) + return None + + +def _relocated_record( + *, + record: DeclarationRecord, + usages_by_declaration: _UsagesByDeclaration, + anchor_sets: _AnchorSets, +) -> DeclarationRecord | None: + required: tuple[ScopeKind, str | None, tuple[str, ...]] | None = _required_placement_for_record( + record=record, usages_by_declaration=usages_by_declaration, anchor_sets=anchor_sets + ) + if required is None: + return None + scope: ScopeKind = required[0] + path: str | None = required[1] + if record.scope is scope and record.owning_path == path: + return None + directory: str = declaration_directory(kind=record.identity.kind, scope=scope, path=path) + roots: set[OwnershipRoot] = { + root for root, _path in anchor_sets.get(record.identity, frozenset()) + } + ownership_root: OwnershipRoot = ( + OwnershipRoot(directory, OwnershipRootKind.GLOBAL) + if scope is ScopeKind.GLOBAL or len(roots) != 1 + else next(iter(roots)) + ) + return replace( + record, + path=f"{directory}/{PurePosixPath(record.path).name}", + scope=scope, + ownership_root=ownership_root, + owning_path=None if scope is ScopeKind.GLOBAL else path, + ) + + def _usages_by_declaration(*, index: ScopeIndex) -> _UsagesByDeclaration: usage_lists: dict[DeclarationIdentity, list[UsageRecord]] = {} for usage in index.usages: @@ -319,17 +415,12 @@ def _message( consumers: str, ) -> str: current_path: str = declaration.owning_path or declaration.ownership_root.path - role_parts: tuple[str, ...] = DECLARATION_ROLE_PARTS[declaration.identity.kind] - if required_scope is ScopeKind.GLOBAL: - target: str = f"top-level {'/'.join(role_parts)}/" - else: - prefix: str = "_" if required_scope is ScopeKind.LOCAL else "" - role: str = "/".join((f"{prefix}{role_parts[0]}", *role_parts[1:])) + "/" - target = ( - f"{required_path}/{role}" - if required_path in _AUTHORED_ROOT_PATHS - else f"{required_path}/{DECLARATION_GROUP_DIRECTORY}/{role}" - ) + directory: str = declaration_directory( + kind=declaration.identity.kind, scope=required_scope, path=required_path + ) + target: str = ( + f"top-level {directory}/" if required_scope is ScopeKind.GLOBAL else f"{directory}/" + ) consumer_label: str = ( "References" if declaration.identity.kind is DeclarationKind.SINGULAR_AUDIT else "Consumers" ) diff --git a/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py b/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py new file mode 100644 index 0000000000..9e4e436ddf --- /dev/null +++ b/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py @@ -0,0 +1,14 @@ +"""Public declaration relocation for a resource move.""" + +from __future__ import annotations + +from sqlbuild.compiler.scopes._helpers.placement import relocate_for_resource_move +from sqlbuild.compiler.scopes.models import DeclarationRecord, ResourceIdentity, ScopeIndex + + +def relocate_declarations_for_move( + *, index: ScopeIndex, resource: ResourceIdentity, destination: str +) -> tuple[DeclarationRecord, ...] | None: + """Return declarations that must move, at their new paths, when a resource moves.""" + + return relocate_for_resource_move(index=index, resource=resource, destination=destination) diff --git a/src/sqlbuild/executor/janitor/_helpers/plan.py b/src/sqlbuild/executor/janitor/_helpers/plan.py index 3e8e4c466f..883c44cb42 100644 --- a/src/sqlbuild/executor/janitor/_helpers/plan.py +++ b/src/sqlbuild/executor/janitor/_helpers/plan.py @@ -8,12 +8,14 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup +from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledProject, CompiledSeed, CompiledSource, ) +from sqlbuild.executor.janitor.constants import QUALIFIED_ORIGIN_PARTS from sqlbuild.executor.janitor.models import JanitorRelationKey @@ -42,6 +44,59 @@ def collect_desired_keys(project: CompiledProject) -> set[JanitorRelationKey]: return keys +def pending_migration_origins( + *, + project: CompiledProject, + relations_by_schema: dict[tuple[str | None, str | None], tuple[RelationInfo, ...]], +) -> dict[JanitorRelationKey, str]: + """Return listed origins of migrations whose destination relation does not exist yet.""" + + listed: dict[tuple[str | None, str | None, str], JanitorRelationKey] = {} + relations: tuple[RelationInfo, ...] + for relations in relations_by_schema.values(): + listed.update({_folded(relation_key(item)): relation_key(item) for item in relations}) + schemas: set[tuple[str | None, str | None]] = collect_target_schemas(project) + pending: dict[JanitorRelationKey, str] = {} + model: CompiledModel + for model in project.models: + raw: object | None = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) + destination: JanitorRelationKey = JanitorRelationKey( + database=model.destination.database, + schema=model.destination.schema, + name=model.destination.name, + ) + if not isinstance(raw, str) or _folded(destination) in listed: + continue + origin: JanitorRelationKey + for origin in _origin_keys(raw=raw, destination=destination, schemas=schemas): + found: JanitorRelationKey | None = listed.get(_folded(origin)) + if found is not None: + pending.setdefault(found, f"pending migration origin for {model.name}") + return pending + + +def _origin_keys( + *, raw: str, destination: JanitorRelationKey, schemas: set[tuple[str | None, str | None]] +) -> tuple[JanitorRelationKey, ...]: + parts: list[str] = [part.strip().strip('"`') for part in raw.split(".")] + if len(parts) == 1: + return tuple( + JanitorRelationKey(database=database, schema=schema, name=parts[0]) + for database, schema in sorted(schemas, key=_schema_sort_key) + ) + return ( + JanitorRelationKey( + database=parts[0] if len(parts) == QUALIFIED_ORIGIN_PARTS else destination.database, + schema=parts[-2], + name=parts[-1], + ), + ) + + +def _folded(key: JanitorRelationKey) -> tuple[str | None, str | None, str]: + return RelationLookup.key(database=key.database, schema=key.schema, name=key.name) + + def collect_target_schemas(project: CompiledProject) -> set[tuple[str | None, str | None]]: """Collect schemas where the compiled project writes relations, one spelling per schema.""" diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py index 2d9db78951..ce81109fec 100644 --- a/src/sqlbuild/executor/janitor/constants.py +++ b/src/sqlbuild/executor/janitor/constants.py @@ -43,3 +43,4 @@ MALFORMED_ARCHIVE_REASON: str = ( "name resembles a janitor archive but does not match the strict archive grammar" ) +QUALIFIED_ORIGIN_PARTS: int = 3 diff --git a/src/sqlbuild/executor/janitor/main/plan.py b/src/sqlbuild/executor/janitor/main/plan.py index 0550f41374..e681589a3a 100644 --- a/src/sqlbuild/executor/janitor/main/plan.py +++ b/src/sqlbuild/executor/janitor/main/plan.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import replace from datetime import UTC, datetime from typing import Any @@ -14,7 +15,11 @@ collect_query_diff_artifact_candidates, gather_janitor_warehouse_facts, ) -from sqlbuild.executor.janitor._helpers.plan import collect_scan_schemas, collect_target_schemas +from sqlbuild.executor.janitor._helpers.plan import ( + collect_scan_schemas, + collect_target_schemas, + pending_migration_origins, +) from sqlbuild.executor.janitor._helpers.schema_planning import classify_target_schemas from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader from sqlbuild.executor.janitor.models import ( @@ -23,6 +28,7 @@ JanitorDirectStatePruneCandidate, JanitorOldNameViewPlanning, JanitorPlan, + JanitorRelationKey, JanitorRelationScope, JanitorSchemaClassification, JanitorWarehouseFacts, @@ -107,12 +113,11 @@ def build_janitor_plan( ) inspection.completed(metadata={"item_count": len(target_schemas)}) old_names: JanitorOldNameViewPlanning - old_names, scope = plan_old_name_view_cleanup( + old_names, scope = _plan_protected_names( adapter=adapter, connection=connection, managed_target_schemas=managed_target_schemas, - relations_by_schema=facts.relations_by_schema, - target_name=project.effective_target_name, + facts=facts, early_drops=early_old_name_view_drops, project=project, scope=scope, @@ -173,3 +178,56 @@ def build_janitor_plan( age_metadata_supported=age_supported, planned_at=now, ) + + +def _plan_protected_names( + *, + adapter: BaseAdapter, + connection: Any, + managed_target_schemas: set[tuple[str | None, str | None]], + facts: JanitorWarehouseFacts, + early_drops: tuple[str, ...], + project: CompiledProject, + scope: JanitorRelationScope, + now: datetime, +) -> tuple[JanitorOldNameViewPlanning, JanitorRelationScope]: + """Plan old-name views, then keep general cleanup away from pending migration origins.""" + + old_names: JanitorOldNameViewPlanning + old_names, scope = plan_old_name_view_cleanup( + adapter=adapter, + connection=connection, + managed_target_schemas=managed_target_schemas, + relations_by_schema=facts.relations_by_schema, + target_name=project.effective_target_name, + early_drops=early_drops, + project=project, + scope=scope, + now=now, + ) + return old_names, _protect_pending_origins( + project=project, facts=facts, scope=scope, old_names=old_names + ) + + +def _protect_pending_origins( + *, + project: CompiledProject, + facts: JanitorWarehouseFacts, + scope: JanitorRelationScope, + old_names: JanitorOldNameViewPlanning, +) -> JanitorRelationScope: + pending: dict[JanitorRelationKey, str] = { + key: reason + for key, reason in pending_migration_origins( + project=project, relations_by_schema=facts.relations_by_schema + ).items() + if key not in old_names.keys + } + if not pending: + return scope + return replace( + scope, + protected_relation_keys=scope.protected_relation_keys | frozenset(pending), + protected_relation_reasons={**(scope.protected_relation_reasons or {}), **pending}, + ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py index 6c2fab0341..5ddec9b493 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py @@ -90,6 +90,8 @@ def build_sqlbuild_plan_output( deferral=deferral, policies=PlannerPolicies(), on_progress=hooks.on_progress, + project_config=environment.discovered_inputs.project_config, + local_config=environment.discovered_inputs.local_config, ) except PlannerInputError: return build_display_only_sqlbuild_plan( diff --git a/src/sqlbuild/lint/_helpers/native.py b/src/sqlbuild/lint/_helpers/native.py index ed30e0bd9d..a2b7a2684e 100644 --- a/src/sqlbuild/lint/_helpers/native.py +++ b/src/sqlbuild/lint/_helpers/native.py @@ -405,7 +405,7 @@ def _top_level_description_value_span(*, header_text: str) -> tuple[int, int] | while index < len(header_text): character: str = header_text[index] if character in _QUOTE_CHARACTERS: - index = _quoted_value_end(text=header_text, start=index) + index = quoted_value_end(text=header_text, start=index) continue if character in closing_delimiters: delimiters.append(closing_delimiters[character]) @@ -429,7 +429,7 @@ def _top_level_description_value_span(*, header_text: str) -> tuple[int, int] | while quote_start < len(header_text) and header_text[quote_start].isspace(): quote_start += 1 if quote_start < len(header_text) and header_text[quote_start] in _QUOTE_CHARACTERS: - quote_end: int = _quoted_value_end(text=header_text, start=quote_start) + quote_end: int = quoted_value_end(text=header_text, start=quote_start) return quote_start + 1, quote_end - 1 index = word_end continue @@ -437,7 +437,9 @@ def _top_level_description_value_span(*, header_text: str) -> tuple[int, int] | return None -def _quoted_value_end(*, text: str, start: int) -> int: +def quoted_value_end(*, text: str, start: int) -> int: + """Return the offset just past the quoted value that opens at start.""" + quote: str = text[start] index: int = start + 1 while index < len(text): diff --git a/src/sqlbuild/lint/main/quoted_value_end.py b/src/sqlbuild/lint/main/quoted_value_end.py new file mode 100644 index 0000000000..f397b2de2a --- /dev/null +++ b/src/sqlbuild/lint/main/quoted_value_end.py @@ -0,0 +1,11 @@ +"""Public end-offset finder for quoted MODEL header values.""" + +from __future__ import annotations + +from sqlbuild.lint._helpers.native import quoted_value_end as _quoted_value_end + + +def quoted_value_end(*, text: str, start: int) -> int: + """Return the offset just past the quoted value that opens at start.""" + + return _quoted_value_end(text=text, start=start) diff --git a/src/sqlbuild/lint/main/scan_interpolation_sites.py b/src/sqlbuild/lint/main/scan_interpolation_sites.py new file mode 100644 index 0000000000..0540c6608a --- /dev/null +++ b/src/sqlbuild/lint/main/scan_interpolation_sites.py @@ -0,0 +1,12 @@ +"""Public lexical scan for SQLBuild interpolation sites in authored SQL.""" + +from __future__ import annotations + +from sqlbuild.lint._helpers.sqlbuild_tokens import neutralize_interpolation +from sqlbuild.lint.models import InterpolationSite + + +def scan_interpolation_sites(*, body: str, dialect: str) -> tuple[InterpolationSite, ...]: + """Return every reference call, macro, and template site outside comments and strings.""" + + return neutralize_interpolation(body=body, dialect=dialect)[1] diff --git a/src/sqlbuild/spec/contracts/_helpers/targets.py b/src/sqlbuild/spec/contracts/_helpers/targets.py index 7be6cdf3fc..aa62d4c54c 100644 --- a/src/sqlbuild/spec/contracts/_helpers/targets.py +++ b/src/sqlbuild/spec/contracts/_helpers/targets.py @@ -112,6 +112,11 @@ def resolve_target_config( if local_target.time_travel_retention_decrease is not None else project_target.time_travel_retention_decrease ), + missing_migration_origin=( + local_target.missing_migration_origin + if local_target.missing_migration_origin is not None + else project_target.missing_migration_origin + ), execution_limits=_merge_execution_limits( project_limits=project_target.execution_limits, local_limits=local_target.execution_limits, diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py index 9559712e85..1643d02f19 100644 --- a/src/sqlbuild/spec/contracts/models.py +++ b/src/sqlbuild/spec/contracts/models.py @@ -14,6 +14,7 @@ EventExportSeverity, FutureCursorAction, MicrobatchLimitAction, + MissingMigrationOriginPolicy, RetentionDecreasePolicy, SourceFreshnessStrategy, SourceFreshnessValueKind, @@ -123,6 +124,7 @@ class TargetConfig: default_table_type: TableType | None = None table_type_downgrade: TableTypeDowngradePolicy = TableTypeDowngradePolicy.REQUIRE_CONFIRMATION time_travel_retention_decrease: RetentionDecreasePolicy = RetentionDecreasePolicy.DENY + missing_migration_origin: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) @@ -146,6 +148,7 @@ class LocalTargetConfig: default_table_type: TableType | None = None table_type_downgrade: TableTypeDowngradePolicy | None = None time_travel_retention_decrease: RetentionDecreasePolicy | None = None + missing_migration_origin: MissingMigrationOriginPolicy | None = None execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) diff --git a/src/sqlbuild/spec/contracts/types.py b/src/sqlbuild/spec/contracts/types.py index 7f822ee5a0..58c0ea6d92 100644 --- a/src/sqlbuild/spec/contracts/types.py +++ b/src/sqlbuild/spec/contracts/types.py @@ -101,6 +101,14 @@ class TableTypeDowngradePolicy(StrEnum): ALLOW = "allow" +class MissingMigrationOriginPolicy(StrEnum): + """Policy for a declared migrate_from whose origin does not exist in a target.""" + + DENY = "deny" + REQUIRE_CONFIRMATION = "require_confirmation" + ALLOW = "allow" + + class RetentionDecreasePolicy(StrEnum): """Policy for lowering live time-travel retention.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py index b2fa50709f..a6c9f07cff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py @@ -107,3 +107,10 @@ class DbtAutoInitE2ETestCase: class DbtCliFlagAmbiguityE2ETestCase: description: str expected_stdout_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DbtMissingOriginE2ETestCase: + description: str + command: tuple[str, ...] + expected_output: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py index b3f7d14d57..3cb72f0257 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py @@ -129,6 +129,26 @@ def write_sqlbuild_defer_target_models(*, project_dir: Path) -> None: ) +def declare_missing_origin_requiring_confirmation(*, project_dir: Path) -> None: + """Give local_only a migrate_from origin that was never built, under require_confirmation.""" + + config_path: Path = project_dir / "sqlbuild_project.toml" + config_path.write_text( + config_path.read_text(encoding="utf-8").replace( + 'adapter = "duckdb"\n', 'adapter = "duckdb"\ndefault_target = "main"\n' + ) + + '\n[targets.main]\nschema = "main"\nmissing_migration_origin = "require_confirmation"\n', + encoding="utf-8", + ) + model_path: Path = project_dir / "models" / "local_only.sql" + model_path.write_text( + model_path.read_text(encoding="utf-8").replace( + "MODEL (\n", "MODEL (\n materialized table,\n migrate_from retired_local_orders,\n", 1 + ), + encoding="utf-8", + ) + + def load_json_stdout(stdout: str) -> dict[str, object]: """Load JSON command output.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py index f9e3c8b924..a2a29bb6a5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py @@ -11,9 +11,11 @@ DbtExecutionCliTestCase, DbtExecutionFailureCliTestCase, DbtExecutionQueryAssertion, + DbtMissingOriginE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.dbt.helpers import ( break_dbt_interop_fact_orders_model, + declare_missing_origin_requiring_confirmation, load_json_stdout, prepare_dbt_interop_project, skip_unless_dbt_is_runnable, @@ -541,3 +543,37 @@ def test_given_failing_dbt_model_when_running_command_then_dependent_sqlbuild_is relation_name: str for relation_name in test_case.expected_absent_relations: assert not table_exists(db_path=db_path, table_name=relation_name), relation_name + + +@pytest.mark.parametrize( + "test_case", + [ + DbtMissingOriginE2ETestCase( + description="dbt build refuses a missing origin that needs confirmation", + command=("dbt", "build", "--select", "tag:sqb_only"), + expected_output="build the affected models with sqb build --allow-missing-migration-origin", + ), + DbtMissingOriginE2ETestCase( + description="dbt run refuses a missing origin that needs confirmation", + command=("dbt", "run", "--select", "local_only"), + expected_output="build the affected models with sqb build --allow-missing-migration-origin", + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_requiring_confirmation_when_running_dbt_then_refuses( + test_case: DbtMissingOriginE2ETestCase, + tmp_path: Path, +) -> None: + skip_unless_dbt_is_runnable() + project_dir: Path = prepare_dbt_interop_project(tmp_path=tmp_path) + declare_missing_origin_requiring_confirmation(project_dir=project_dir) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir, input_text="" + ) + + assert result.returncode == 1, result.stdout + result.stderr + assert test_case.expected_output in result.stdout + result.stderr + assert "Pass --allow-missing-migration-origin" not in result.stdout + result.stderr + assert not table_exists(db_path=project_dir / "dbt_interop.duckdb", table_name="local_only") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py new file mode 100644 index 0000000000..910f4adea4 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py @@ -0,0 +1,106 @@ +"""Test case types for `sqb rename` and `sqb mv` e2e tests.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class ModelRefactorE2ETestCase: + description: str + command: tuple[str, ...] + expected_removed: str + expected_file: str + expected_fragments: dict[str, tuple[str, ...]] + expected_relation: str + extra_files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ModelMigrationE2ETestCase: + description: str + materialized: str + udf: bool + expected_declaration: str + expected_old_name_type: str + expected_order_ids: tuple[int, ...] + extra_files: dict[str, str] = field(default_factory=dict) + removed_files: tuple[str, ...] = () + stripped_fingerprints: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ColumnRenameE2ETestCase: + description: str + command: tuple[str, ...] + extra_files: dict[str, str] = field(default_factory=dict) + expected_fragments: dict[str, tuple[str, ...]] = field(default_factory=dict) + expected_columns: dict[str, tuple[str, ...]] = field(default_factory=dict) + expected_manual: tuple[str, ...] = () + + +@dataclass(frozen=True) +class RefusedRefactorE2ETestCase: + description: str + command: tuple[str, ...] + extra_files: dict[str, str] + expected_status: str + expected_reason: str + expected_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class ColumnMigrationE2ETestCase: + description: str + expected_declaration: str + expected_columns: tuple[str, ...] + expected_order_ids: tuple[int, ...] + + +@dataclass(frozen=True) +class DryRunE2ETestCase: + description: str + command: tuple[str, ...] + expected_paths: frozenset[str] + + +@dataclass(frozen=True) +class CombinedRenameE2ETestCase: + description: str + expected_declarations: tuple[str, ...] + expected_new_columns: tuple[str, ...] + expected_old_columns: tuple[str, ...] + + +@dataclass(frozen=True) +class MissingOriginE2ETestCase: + description: str + target_settings: str + build_args: tuple[str, ...] + expected_exit: int + expected_output: str + expected_relation_type: str | None + expected_columns: tuple[str, ...] = () + + +@dataclass(frozen=True) +class DeclarationMoveE2ETestCase: + description: str + model: str + extra_files: dict[str, str] + destination: str + expected_moved: dict[str, str] + + +@dataclass(frozen=True) +class HiddenOriginE2ETestCase: + description: str + target_settings: str + expected_output: str + + +@dataclass(frozen=True) +class PendingOriginJanitorE2ETestCase: + description: str + janitor_args: tuple[str, ...] + expected_order_ids: tuple[int, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py new file mode 100644 index 0000000000..78b1284e7a --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -0,0 +1,400 @@ +"""Project builders and warehouse readers for `sqb rename` and `sqb mv` e2e tests.""" + +from __future__ import annotations + +import base64 +import json +import subprocess +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +DATABASE_FILE: str = "orders.duckdb" +SCHEMA: str = "analytics" + + +def project_toml(*, target_settings: str = "") -> str: + """Return the project config, with extra settings for the dev target.""" + + return ( + 'name = "orders_project"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + f'[connection]\ndatabase = "{DATABASE_FILE}"\n\n' + f'[targets.dev]\nschema = "{SCHEMA}"\n{target_settings}\n' + "[janitor]\nenabled = true\n" + ) + + +_PROJECT_TOML: str = project_toml() +_SOURCES_YML: str = ( + "sources:\n" + " - name: raw_orders\n" + " schema: main\n" + " table: raw_orders\n" + " columns:\n" + " - name: order_id\n" + " - name: customer_id\n" + " - name: amount\n" + " - name: order_date\n" +) +STG_ORDERS: str = ( + "MODEL (\n materialized view,\n);\n\n" + "SELECT\n order_id,\n customer_id,\n amount,\n order_date\n" + 'FROM __source("raw_orders")\n' +) +FACT_ORDERS: str = ( + "MODEL (\n materialized table,\n);\n\n" + "SELECT\n o.order_id,\n o.customer_id,\n o.amount,\n o.order_date\n" + 'FROM __ref("stg_orders") AS o\n' +) +CUSTOMER_TOTALS: str = ( + "MODEL (\n materialized table,\n);\n\n" + "SELECT\n customer_id,\n SUM(amount) AS total_amount\n" + 'FROM __ref("fact_orders")\nGROUP BY customer_id\n' +) +FACT_ORDERS_TEST: str = ( + "TEST();\n\n" + "WITH\n" + "__ref__stg_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, 25 AS amount, " + "CAST('2026-01-01' AS TIMESTAMP) AS order_date\n" + "),\n" + "__expected__fact_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, 25 AS amount, " + "CAST('2026-01-01' AS TIMESTAMP) AS order_date\n" + ")\n" + "SELECT 1\n" +) +BASE_FILES: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT_TOML, + "sources/raw.yml": _SOURCES_YML, + "models/staging/stg_orders.sql": STG_ORDERS, + "models/marts/fact_orders.sql": FACT_ORDERS, + "models/marts/customer_totals.sql": CUSTOMER_TOTALS, + "tests/unit/test_fact_orders.sql": FACT_ORDERS_TEST, +} +UDF_FILE: tuple[str, str] = ( + "functions/sql/udf__is_large_order.sql", + "FUNCTION (\n arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\namount >= 100\n", +) +CENTS_MACRO: dict[str, str] = { + "models/staging/_sqlbuild/_macros/cents.py": ( + 'def to_cents(expression: str) -> str:\n return f"({expression}) * 100"\n' + ), + "models/staging/stg_order_cents.sql": ( + "MODEL (\n materialized view,\n);\n\n" + 'SELECT\n order_id,\n @to_cents("amount") AS amount_cents\nFROM __ref("stg_orders")\n' + ), +} +ORDER_REFUNDS: dict[str, str] = { + "models/staging/stg_order_refunds.sql": ( + "MODEL (\n materialized view,\n);\n\n" + 'SELECT\n order_id,\n @to_cents("amount") AS refund_cents\nFROM __ref("stg_orders")\n' + ), +} +SPLIT_CENTS_MACROS: dict[str, str] = { + **CENTS_MACRO, + "models/staging/_sqlbuild/_macros/cents.py": ( + CENTS_MACRO["models/staging/_sqlbuild/_macros/cents.py"] + + '\n\ndef to_dollars(expression: str) -> str:\n return f"({expression}) / 100"\n' + ), + "models/staging/stg_order_refunds.sql": ORDER_REFUNDS[ + "models/staging/stg_order_refunds.sql" + ].replace("to_cents", "to_dollars"), +} + + +def order_amount_union(*, first: str, second: str) -> dict[str, str]: + """Return a view unioning amount from two models, and a view reading its output.""" + + return { + "models/marts/order_amounts.sql": ( + "MODEL (\n materialized view,\n);\n\n" + f'SELECT amount FROM __ref("{first}")\nUNION ALL\n' + f'SELECT amount FROM __ref("{second}")\n' + ), + "models/marts/order_amount_reads.sql": ( + 'MODEL (\n materialized view,\n);\n\nSELECT amount FROM __ref("order_amounts")\n' + ), + } + + +ORDER_EXPORT: dict[str, str] = { + "models/marts/order_export.sql": ( + 'MODEL (\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' + ), +} +CUSTOMER_DECLARATIONS: dict[str, str] = { + "seeds/customers.csv": "customer_id,customer_name\n10,Ada\n11,Bo\n", + "seeds/customers.yml": ( + "seeds:\n" + " - name: customers\n" + " columns:\n" + " - name: customer_id\n" + " type: INTEGER\n" + " audits:\n" + " - relationships:\n" + ' to: __ref("stg_orders")\n' + " field: customer_id\n" + " - name: customer_name\n" + " type: VARCHAR\n" + ), + "sources/customers.yml": ( + "sources:\n" + " - name: raw_customers\n" + ' expression: "SELECT 10 AS customer_id UNION ALL SELECT 11"\n' + " columns:\n" + " - name: customer_id\n" + " audits:\n" + " - relationships: {to: '__ref(\"stg_orders\")', field: customer_id}\n" + ), +} +ORDER_SHAPE: dict[str, str] = { + "models/marts/_sqlbuild/_schemas/order_shape.sql": ( + "SCHEMA (\n name order_shape,\n columns (\n customer_id (audits [relationships " + '(to __ref("stg_orders"), field customer_id)]),\n ),\n);\n' + ), + "models/marts/fact_orders.sql": FACT_ORDERS.replace( + "materialized table,", "materialized table,\n model_schema order_shape," + ), +} +ORDERS_MACRO: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT_TOML + "\n[references]\nenforce_explicit = false\n", + "models/staging/_sqlbuild/_macros/orders.py": ( + "def staged_orders() -> str:\n return '__ref(\"stg_orders\")'\n" + ), + "models/staging/stg_order_count.sql": ( + "MODEL (\n materialized view,\n);\n\n" + "SELECT COUNT(*) AS order_count\nFROM @staged_orders()\n" + ), +} +_INCREMENTAL_HEADER: str = ( + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_id,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + " cursor_inputs (\n stg_orders order_date,\n ),\n" +) + + +_HISTORY_HEADERS: dict[str, str] = { + "table": " materialized table,\n", + "view": " materialized view,\n", + "incremental": _INCREMENTAL_HEADER, +} +_UDF_COLUMN: dict[bool, str] = { + True: ',\n __udf("udf__is_large_order")(amount) AS is_large', + False: "", +} +_UDF_FILES: dict[bool, dict[str, str]] = {True: {UDF_FILE[0]: UDF_FILE[1]}, False: {}} + + +def order_history_files(*, materialized: str, udf: bool) -> dict[str, str]: + """Return an order_history model, optionally calling a UDF discovery cannot fingerprint.""" + + return { + "models/marts/order_history.sql": ( + f"MODEL (\n{_HISTORY_HEADERS[materialized]});\n\n" + f"SELECT\n order_id,\n amount,\n order_date{_UDF_COLUMN[udf]}\n" + 'FROM __ref("stg_orders")\n' + ), + **_UDF_FILES[udf], + } + + +def write_orders_project( + *, tmp_path: Path, files: Mapping[str, str] | None = None, raw_order_count: int = 3 +) -> Path: + """Write the base orders project plus overrides and load raw orders.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="orders_project", repo_files={**BASE_FILES, **(files or {})} + ) + load_raw_orders(project_dir=project_dir, order_ids=tuple(range(1, raw_order_count + 1))) + return project_dir + + +def order_history_copy() -> dict[str, str]: + """Return a second model with exactly the order_history definition.""" + + return { + "models/marts/order_history_copy.sql": order_history_files( + materialized="incremental", udf=False + )["models/marts/order_history.sql"] + } + + +def strip_migration_fingerprints(*, project_dir: Path, models: tuple[str, ...]) -> None: + """Make built models look like ones built before rename matching recorded fingerprints.""" + + import duckdb + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(project_dir / DATABASE_FILE)) + try: + rows: list[tuple[Any, ...]] = connection.execute( + f"SELECT node_name, metadata_json_b64 FROM {SCHEMA}._sqlbuild_fingerprints " + "WHERE list_contains(?, node_name)", + [list(models)], + ).fetchall() + model: str + encoded: str + for model, encoded in rows: + metadata: dict[str, Any] = json.loads(base64.b64decode(encoded)) + _ = metadata.pop("migration_fingerprint", None) + _ = connection.execute( + f"UPDATE {SCHEMA}._sqlbuild_fingerprints SET metadata_json_b64 = ? " + "WHERE node_name = ? AND metadata_json_b64 = ?", + [base64.b64encode(json.dumps(metadata).encode()).decode(), model, encoded], + ) + finally: + connection.close() + + +def drop_view_outside_sqlbuild(*, project_dir: Path, name: str) -> None: + """Drop a built view directly in the warehouse, leaving SQLBuild's history in place.""" + + import duckdb + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(project_dir / DATABASE_FILE)) + try: + _ = connection.execute(f"DROP VIEW {SCHEMA}.{name}") + finally: + connection.close() + + +def declare_missing_column_origin(*, project_dir: Path) -> None: + """Add a revenue column that declares migrate_from a column the table never had.""" + + path: Path = project_dir / "models/marts/order_history.sql" + contents: str = path.read_text(encoding="utf-8") + _ = path.write_text( + contents.replace( + "MODEL (\n", "MODEL (\n columns (revenue (migrate_from gross_amount)),\n", 1 + ).replace(" order_date\n", " order_date,\n amount AS revenue\n", 1), + encoding="utf-8", + ) + + +def existing_files(*, project_dir: Path, paths: tuple[str, ...]) -> tuple[str, ...]: + """Return the given project-relative paths that exist, sorted.""" + + present: frozenset[str] = frozenset( + path.relative_to(project_dir).as_posix() for path in project_dir.rglob("*.*") + ) + return tuple(sorted(frozenset(paths) & present)) + + +def remove_files(*, project_dir: Path, paths: tuple[str, ...]) -> None: + """Delete project files by hand, as an author removing a model would.""" + + path: str + for path in paths: + (project_dir / path).unlink() + + +def load_raw_orders(*, project_dir: Path, order_ids: tuple[int, ...]) -> None: + """Replace the raw orders with one order per id, dated January of that day.""" + + values: str = ", ".join( + f"({order_id}, {10 + order_id % 2}, {order_id * 50}, TIMESTAMP '2026-01-{order_id:02d}')" + for order_id in order_ids + ) + execute_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "CREATE OR REPLACE TABLE main.raw_orders AS SELECT * FROM (VALUES " + f"{values}) AS t(order_id, customer_id, amount, order_date)" + ), + ) + + +def sqb(project_dir: Path, *args: str) -> subprocess.CompletedProcess[str]: + """Run one sqb command without colour.""" + + return run_sqb(command=("--no-color", *args), project_dir=project_dir) + + +def sqb_json(project_dir: Path, *args: str) -> tuple[int, dict[str, Any]]: + """Run one sqb command with --json and parse its stdout.""" + + result: subprocess.CompletedProcess[str] = sqb(project_dir, *args, "--json") + return result.returncode, json.loads(result.stdout) + + +def project_text(project_dir: Path) -> dict[str, str]: + """Return every authored project file keyed by its relative path.""" + + paths: tuple[Path, ...] = ( + project_dir / "sqlbuild_project.toml", + *project_dir.glob("sources/*.yml"), + *project_dir.glob("seeds/*.yml"), + *project_dir.glob("models/**/*.sql"), + *project_dir.glob("models/**/*.py"), + *project_dir.glob("functions/**/*.sql"), + *project_dir.glob("tests/**/*.sql"), + ) + return { + path.relative_to(project_dir).as_posix(): path.read_text(encoding="utf-8") + for path in sorted(paths) + } + + +def fragments_present(*, files: dict[str, str], expected: dict[str, tuple[str, ...]]) -> bool: + """Return whether every expected fragment appears in its file.""" + + found: list[bool] = [] + path: str + fragments: tuple[str, ...] + for path, fragments in expected.items(): + found.extend(fragment in files[path] for fragment in fragments) + return all(found) + + +def relation_type(*, project_dir: Path, name: str) -> str | None: + """Return the information_schema table type of one relation, or None.""" + + rows: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT table_type FROM information_schema.tables " + f"WHERE table_schema = '{SCHEMA}' AND table_name = '{name}'" + ), + ) + return next((str(row[0]) for row in rows), None) + + +def relation_columns(*, project_dir: Path, name: str) -> tuple[str, ...]: + """Return the column names of one relation in ordinal order.""" + + return tuple( + str(row[0]) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT column_name FROM information_schema.columns " + f"WHERE table_schema = '{SCHEMA}' AND table_name = '{name}' " + "ORDER BY ordinal_position" + ), + ) + ) + + +def order_ids(*, project_dir: Path, name: str) -> tuple[int, ...]: + """Return the order ids stored in one relation.""" + + return tuple( + int(row[0]) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=f"SELECT order_id FROM {SCHEMA}.{name} ORDER BY order_id", + ) + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py new file mode 100644 index 0000000000..f890b505b4 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py @@ -0,0 +1,72 @@ +"""CLI e2e coverage for the janitor keeping the origin of a pending model migration.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + PendingOriginJanitorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + load_raw_orders, + order_history_files, + order_ids, + relation_type, + sqb, + write_orders_project, +) + +_PENDING_REASON: str = "analytics.order_history pending migration origin for order_ledger" + + +@pytest.mark.parametrize( + "test_case", + [ + PendingOriginJanitorE2ETestCase( + description="janitor keeps the origin until the rename is migrated", + janitor_args=("janitor", "--retention-days", "0", "--auto-approve"), + expected_order_ids=(1, 2, 3, 4), + ), + ], + ids=lambda case: case.description, +) +def test_given_pending_rename_when_janitor_runs_then_origin_survives_until_migrated( + tmp_path: Path, test_case: PendingOriginJanitorE2ETestCase +) -> None: + project_dir: Path = write_orders_project( + tmp_path=tmp_path, files=order_history_files(materialized="incremental", udf=False) + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + renamed: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "model:order_history", "order_ledger" + ) + + pending: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.janitor_args) + origin_after_pending: str | None = relation_type(project_dir=project_dir, name="order_history") + load_raw_orders(project_dir=project_dir, order_ids=(3, 4)) + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + migrated: tuple[int, ...] = order_ids(project_dir=project_dir, name="order_ledger") + settled: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.janitor_args) + old_name_after_settled: str | None = relation_type( + project_dir=project_dir, name="order_history" + ) + dropped: subprocess.CompletedProcess[str] = sqb( + project_dir, *test_case.janitor_args, "--drop-old-name-view", "order_history" + ) + + assert first.returncode == 0, first.stdout + first.stderr + assert renamed.returncode == 0, renamed.stdout + renamed.stderr + assert pending.returncode == 0, pending.stdout + pending.stderr + assert _PENDING_REASON in pending.stdout + assert origin_after_pending == "BASE TABLE" + assert built.returncode == 0, built.stdout + built.stderr + assert migrated == test_case.expected_order_ids + assert settled.returncode == 0, settled.stdout + settled.stderr + assert "pending migration origin" not in settled.stdout + assert "analytics.order_history -> model:order_ledger" in settled.stdout + assert old_name_after_settled == "VIEW" + assert dropped.returncode == 0, dropped.stdout + dropped.stderr + assert relation_type(project_dir=project_dir, name="order_history") is None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py new file mode 100644 index 0000000000..9bedf27127 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py @@ -0,0 +1,179 @@ +"""CLI e2e coverage for the missing_migration_origin target policy.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + HiddenOriginE2ETestCase, + MissingOriginE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + declare_missing_column_origin, + drop_view_outside_sqlbuild, + order_history_files, + project_toml, + relation_columns, + relation_type, + sqb, + write_orders_project, +) + +_DENY: str = 'missing_migration_origin = "deny"\n' +_CONFIRM: str = 'missing_migration_origin = "require_confirmation"\n' + + +@pytest.mark.parametrize( + "test_case", + [ + MissingOriginE2ETestCase( + description="fresh target builds with a warning by default", + target_settings="", + build_args=(), + expected_exit=0, + expected_output="migrate_from origin analytics.stg_orders does not exist", + expected_relation_type="VIEW", + ), + MissingOriginE2ETestCase( + description="deny stops the build", + target_settings=_DENY, + build_args=(), + expected_exit=1, + expected_output="error[M102]", + expected_relation_type=None, + ), + MissingOriginE2ETestCase( + description="require_confirmation stops an unconfirmed build", + target_settings=_CONFIRM, + build_args=(), + expected_exit=1, + expected_output="--allow-missing-migration-origin", + expected_relation_type=None, + ), + MissingOriginE2ETestCase( + description="require_confirmation builds once confirmed", + target_settings=_CONFIRM, + build_args=("--allow-missing-migration-origin",), + expected_exit=0, + expected_output="does not exist in this target", + expected_relation_type="VIEW", + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_model_on_fresh_target_when_building_then_policy_decides( + tmp_path: Path, test_case: MissingOriginE2ETestCase +) -> None: + """A rename's migrate_from finds no old relation on a target that never built it.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, + files={"sqlbuild_project.toml": project_toml(target_settings=test_case.target_settings)}, + ) + + renamed: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "model:stg_orders", "stg_order_lines" + ) + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build", *test_case.build_args) + + assert renamed.returncode == 0, renamed.stdout + renamed.stderr + assert built.returncode == test_case.expected_exit, built.stdout + built.stderr + assert test_case.expected_output in built.stdout + built.stderr + assert ( + relation_type(project_dir=project_dir, name="stg_order_lines") + == test_case.expected_relation_type + ) + + +@pytest.mark.parametrize( + "test_case", + [ + MissingOriginE2ETestCase( + description="missing origin column is allowed by default", + target_settings="", + build_args=(), + expected_exit=0, + expected_output="column gross_amount does not exist", + expected_relation_type="BASE TABLE", + expected_columns=("order_id", "amount", "order_date", "revenue"), + ), + MissingOriginE2ETestCase( + description="missing origin column stops the build under deny", + target_settings=_DENY, + build_args=(), + expected_exit=1, + expected_output="error[M109]", + expected_relation_type="BASE TABLE", + expected_columns=("order_id", "amount", "order_date"), + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_column_when_building_then_policy_decides( + tmp_path: Path, test_case: MissingOriginE2ETestCase +) -> None: + """A column migrate_from naming a column the table never had follows the same policy.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, + files={ + **order_history_files(materialized="incremental", udf=False), + "sqlbuild_project.toml": project_toml(target_settings=test_case.target_settings), + }, + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + declare_missing_column_origin(project_dir=project_dir) + + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert first.returncode == 0, first.stdout + first.stderr + assert built.returncode == test_case.expected_exit, built.stdout + built.stderr + assert test_case.expected_output in built.stdout + built.stderr + assert relation_columns(project_dir=project_dir, name="order_history") == ( + test_case.expected_columns + ) + + +@pytest.mark.parametrize( + "test_case", + [ + HiddenOriginE2ETestCase( + description="an origin built here but dropped outside SQLBuild stops under allow", + target_settings="", + expected_output="was built in this target but is not visible", + ), + HiddenOriginE2ETestCase( + description="require_confirmation cannot confirm past a hidden origin", + target_settings=_CONFIRM, + expected_output="error[M102]", + ), + ], + ids=lambda case: case.description, +) +def test_given_built_origin_not_visible_when_building_then_stops_regardless_of_policy( + tmp_path: Path, test_case: HiddenOriginE2ETestCase +) -> None: + """Recorded build history means the origin should exist, so its absence is an error.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, + files={"sqlbuild_project.toml": project_toml(target_settings=test_case.target_settings)}, + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + drop_view_outside_sqlbuild(project_dir=project_dir, name="stg_orders") + renamed: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "model:stg_orders", "stg_order_lines" + ) + + built: subprocess.CompletedProcess[str] = sqb( + project_dir, "build", "--allow-missing-migration-origin" + ) + + assert first.returncode == 0, first.stdout + first.stderr + assert renamed.returncode == 0, renamed.stdout + renamed.stderr + assert built.returncode == 1, built.stdout + built.stderr + assert test_case.expected_output in built.stdout + built.stderr + assert relation_type(project_dir=project_dir, name="stg_order_lines") is None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_move_declarations.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_move_declarations.py new file mode 100644 index 0000000000..1305866406 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_move_declarations.py @@ -0,0 +1,88 @@ +"""CLI e2e coverage for `sqb mv` carrying scoped declarations to where placement requires.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + DeclarationMoveE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + CENTS_MACRO, + ORDER_REFUNDS, + ORDER_SHAPE, + existing_files, + relation_type, + sqb, + write_orders_project, +) + +_CENTS: str = "models/staging/_sqlbuild/_macros/cents.py" +_PRIORITY_ENUM: dict[str, str] = { + "models/marts/_sqlbuild/_enums/priority.sql": ( + "ENUM (\n name priority,\n members (LOW 1, HIGH 3),\n);\n" + ), + "models/marts/fact_orders.sql": ORDER_SHAPE["models/marts/fact_orders.sql"].replace( + "SELECT", 'SELECT\n @enum("priority").LOW AS priority,', 1 + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationMoveE2ETestCase( + description="declarations only the moved model uses move with it", + model="fact_orders", + extra_files={**ORDER_SHAPE, **_PRIORITY_ENUM}, + destination="models/reporting/", + expected_moved={ + "models/marts/_sqlbuild/_schemas/order_shape.sql": ( + "models/reporting/_sqlbuild/_schemas/order_shape.sql" + ), + "models/marts/_sqlbuild/_enums/priority.sql": ( + "models/reporting/_sqlbuild/_enums/priority.sql" + ), + }, + ), + DeclarationMoveE2ETestCase( + description="a macro still used at the old folder moves up to the shared owner", + model="stg_order_cents", + extra_files={**CENTS_MACRO, **ORDER_REFUNDS}, + destination="models/staging/eu/", + expected_moved={_CENTS: "models/staging/_sqlbuild/macros/cents.py"}, + ), + DeclarationMoveE2ETestCase( + description="a macro shared across the models root moves to the top-level role", + model="stg_order_cents", + extra_files={**CENTS_MACRO, **ORDER_REFUNDS}, + destination="models/marts/", + expected_moved={_CENTS: "macros/cents.py"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_scoped_declarations_when_moving_model_then_they_follow_placement_rules( + tmp_path: Path, test_case: DeclarationMoveE2ETestCase +) -> None: + project_dir: Path = write_orders_project(tmp_path=tmp_path, files=test_case.extra_files) + model: str = test_case.model + + moved: subprocess.CompletedProcess[str] = sqb( + project_dir, "mv", f"model:{model}", test_case.destination + ) + compiled: subprocess.CompletedProcess[str] = sqb(project_dir, "compile") + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert moved.returncode == 0, moved.stdout + moved.stderr + assert existing_files( + project_dir=project_dir, + paths=(*test_case.expected_moved, *test_case.expected_moved.values()), + ) == tuple(sorted(test_case.expected_moved.values())), moved.stdout + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + assert "error[S0" not in compiled.stdout + compiled.stderr + assert built.returncode == 0, built.stdout + built.stderr + assert relation_type(project_dir=project_dir, name=model) is not None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py new file mode 100644 index 0000000000..da5f81b6ae --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py @@ -0,0 +1,311 @@ +"""CLI e2e coverage for `sqb rename column:` and for refactorings that must not write.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + ColumnMigrationE2ETestCase, + ColumnRenameE2ETestCase, + CombinedRenameE2ETestCase, + RefusedRefactorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + CENTS_MACRO, + CUSTOMER_DECLARATIONS, + ORDER_EXPORT, + ORDER_SHAPE, + ORDERS_MACRO, + SPLIT_CENTS_MACROS, + fragments_present, + load_raw_orders, + order_amount_union, + order_history_files, + order_ids, + project_text, + relation_columns, + sqb, + sqb_json, + write_orders_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnRenameE2ETestCase( + description="one step keeps downstream output names", + command=("rename", "column:stg_orders.amount", "revenue"), + expected_fragments={ + "models/staging/stg_orders.sql": (" amount AS revenue,",), + "models/marts/fact_orders.sql": (" o.revenue AS amount,",), + "tests/unit/test_fact_orders.sql": ("25 AS revenue",), + }, + expected_columns={ + "stg_orders": ("order_id", "customer_id", "revenue", "order_date"), + "fact_orders": ("order_id", "customer_id", "amount", "order_date"), + }, + ), + ColumnRenameE2ETestCase( + description="cascade renames pass-through outputs downstream", + command=("rename", "column:stg_orders.amount", "revenue", "--cascade"), + expected_fragments={ + "models/marts/fact_orders.sql": (" o.revenue,",), + "models/marts/customer_totals.sql": ("SUM(revenue) AS total_amount",), + }, + expected_columns={ + "fact_orders": ("order_id", "customer_id", "revenue", "order_date"), + "customer_totals": ("customer_id", "total_amount"), + }, + ), + ColumnRenameE2ETestCase( + description="seed and source relationships follow the column", + command=("rename", "column:stg_orders.customer_id", "buyer_id"), + extra_files=CUSTOMER_DECLARATIONS, + expected_fragments={ + "seeds/customers.yml": ("field: buyer_id",), + "sources/customers.yml": ("field: buyer_id}",), + "models/marts/fact_orders.sql": (" o.buyer_id AS customer_id,",), + }, + expected_columns={ + "stg_orders": ("order_id", "buyer_id", "amount", "order_date"), + }, + ), + ColumnRenameE2ETestCase( + description="reusable schema relationships follow the column", + command=("rename", "column:stg_orders.customer_id", "buyer_id"), + extra_files=ORDER_SHAPE, + expected_fragments={ + "models/marts/_sqlbuild/_schemas/order_shape.sql": ("field buyer_id)",), + }, + expected_columns={ + "fact_orders": ("order_id", "customer_id", "amount", "order_date"), + }, + ), + ColumnRenameE2ETestCase( + description="cascade stops at a later set-operation branch", + command=("rename", "column:fact_orders.amount", "revenue", "--cascade"), + extra_files=order_amount_union(first="stg_orders", second="fact_orders"), + expected_fragments={ + "models/marts/order_amounts.sql": ('SELECT revenue FROM __ref("fact_orders")',), + "models/marts/order_amount_reads.sql": ("SELECT amount FROM",), + }, + expected_columns={ + "order_amounts": ("amount",), + "order_amount_reads": ("amount",), + }, + ), + ColumnRenameE2ETestCase( + description="cascade follows the first set-operation branch", + command=("rename", "column:fact_orders.amount", "revenue", "--cascade"), + extra_files=order_amount_union(first="fact_orders", second="stg_orders"), + expected_fragments={ + "models/marts/order_amounts.sql": ( + 'SELECT revenue FROM __ref("fact_orders")', + 'SELECT amount FROM __ref("stg_orders")', + ), + "models/marts/order_amount_reads.sql": ("SELECT revenue FROM",), + }, + expected_columns={ + "order_amounts": ("revenue",), + "order_amount_reads": ("revenue",), + }, + ), + ColumnRenameE2ETestCase( + description="cascade follows SELECT star consumers", + command=("rename", "column:fact_orders.amount", "revenue", "--cascade"), + extra_files=ORDER_EXPORT, + expected_fragments={"models/marts/customer_totals.sql": ("SUM(revenue)",)}, + expected_columns={ + "fact_orders": ("order_id", "customer_id", "revenue", "order_date"), + "order_export": ("order_id", "customer_id", "revenue", "order_date"), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_column_rename_when_applied_then_project_tests_and_builds( + tmp_path: Path, test_case: ColumnRenameE2ETestCase +) -> None: + """Column references, fixtures, and outputs change together, and the project still runs.""" + + project_dir: Path = write_orders_project(tmp_path=tmp_path, files=test_case.extra_files) + + code: int + payload: dict[str, Any] + code, payload = sqb_json(project_dir, *test_case.command) + tested: subprocess.CompletedProcess[str] = sqb(project_dir, "test") + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert code == 0, payload + assert payload["status"] == "applied" + assert tuple(item["path"] for item in payload["manual"]) == test_case.expected_manual + files: dict[str, str] = project_text(project_dir) + assert fragments_present(files=files, expected=test_case.expected_fragments), files + assert tested.returncode == 0, tested.stdout + tested.stderr + assert built.returncode == 0, built.stdout + built.stderr + assert { + name: relation_columns(project_dir=project_dir, name=name) + for name in test_case.expected_columns + } == test_case.expected_columns + + +@pytest.mark.parametrize( + "test_case", + [ + RefusedRefactorE2ETestCase( + description="move needing to split a declaration file", + command=("mv", "model:stg_order_cents", "models/marts/"), + extra_files=SPLIT_CENTS_MACROS, + expected_status="refused", + expected_reason="would take macro:to_dollars along, which must stay in", + expected_paths=("models/staging/_sqlbuild/_macros/cents.py",), + ), + RefusedRefactorE2ETestCase( + description="rename onto an existing model", + command=("rename", "model:stg_orders", "fact_orders"), + extra_files={}, + expected_status="refused", + expected_reason="model:fact_orders already exists", + expected_paths=("models/marts/fact_orders.sql",), + ), + RefusedRefactorE2ETestCase( + description="column used inside macro-generated SQL", + command=("rename", "column:stg_orders.amount", "revenue"), + extra_files=CENTS_MACRO, + expected_status="refused", + expected_reason="referenced in SQL a macro generates", + expected_paths=("models/staging/stg_order_cents.sql",), + ), + RefusedRefactorE2ETestCase( + description="SELECT star consumer without cascade", + command=("rename", "column:fact_orders.amount", "revenue"), + extra_files=ORDER_EXPORT, + expected_status="refused", + expected_reason="rerun with --cascade", + expected_paths=("models/marts/order_export.sql",), + ), + RefusedRefactorE2ETestCase( + description="model reference produced by a macro", + command=("rename", "model:stg_orders", "stg_order_lines"), + extra_files=ORDERS_MACRO, + expected_status="refused", + expected_reason="reference to stg_orders produced by a macro", + expected_paths=("models/staging/stg_order_count.sql",), + ), + ], + ids=lambda case: case.description, +) +def test_given_unsafe_refactor_when_running_then_no_file_changes( + tmp_path: Path, test_case: RefusedRefactorE2ETestCase +) -> None: + """Refusals and failed verification exit 1, explain why, and leave every file untouched.""" + + project_dir: Path = write_orders_project(tmp_path=tmp_path, files=test_case.extra_files) + before: dict[str, str] = project_text(project_dir) + + code: int + payload: dict[str, Any] + code, payload = sqb_json(project_dir, *test_case.command) + text: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.command) + + assert code == 1 + assert payload["status"] == test_case.expected_status + assert test_case.expected_reason in json.dumps(payload) + assert ( + tuple(sorted({item["path"] for item in (*payload["manual"], *payload["blocking"])})) + == test_case.expected_paths + ) + assert text.returncode == 1 + assert "no files changed" in text.stdout + assert project_text(project_dir) == before + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnMigrationE2ETestCase( + description="incremental column renamed in place", + expected_declaration="revenue (migrate_from amount)", + expected_columns=("order_id", "revenue", "order_date"), + expected_order_ids=(1, 2, 3, 4), + ) + ], + ids=lambda case: case.description, +) +def test_given_built_incremental_when_renaming_column_then_build_keeps_history( + tmp_path: Path, test_case: ColumnMigrationE2ETestCase +) -> None: + """The rename declares the column migration, so the next build renames it in place.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, files=order_history_files(materialized="incremental", udf=False) + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + result: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "column:order_history.amount", "revenue" + ) + header: str = (project_dir / "models/marts/order_history.sql").read_text(encoding="utf-8") + load_raw_orders(project_dir=project_dir, order_ids=(3, 4)) + second: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert first.returncode == 0, first.stdout + first.stderr + assert result.returncode == 0, result.stdout + result.stderr + assert test_case.expected_declaration in header, header + assert second.returncode == 0, second.stdout + second.stderr + assert relation_columns(project_dir=project_dir, name="order_history") == ( + test_case.expected_columns + ) + assert order_ids(project_dir=project_dir, name="order_history") == ( + test_case.expected_order_ids + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CombinedRenameE2ETestCase( + description="table renamed, then its column, before one build", + expected_declarations=("migrate_from order_history", "revenue (migrate_from amount)"), + expected_new_columns=("order_id", "revenue", "order_date"), + expected_old_columns=("order_id", "amount", "order_date"), + ) + ], + ids=lambda case: case.description, +) +def test_given_renamed_table_when_renaming_its_column_then_build_keeps_both( + tmp_path: Path, test_case: CombinedRenameE2ETestCase +) -> None: + """A renamed table declares migrate_from, so its column rename is declared too.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, files=order_history_files(materialized="table", udf=False) + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + model: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "model:order_history", "order_ledger" + ) + column: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "column:order_ledger.amount", "revenue" + ) + header: str = (project_dir / "models/marts/order_ledger.sql").read_text(encoding="utf-8") + second: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert first.returncode == 0, first.stdout + first.stderr + assert model.returncode == 0, model.stdout + model.stderr + assert column.returncode == 0, column.stdout + column.stderr + assert all(declaration in header for declaration in test_case.expected_declarations), header + assert second.returncode == 0, second.stdout + second.stderr + assert relation_columns(project_dir=project_dir, name="order_ledger") == ( + test_case.expected_new_columns + ) + assert relation_columns(project_dir=project_dir, name="order_history") == ( + test_case.expected_old_columns + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py new file mode 100644 index 0000000000..3e82dbbd6c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py @@ -0,0 +1,251 @@ +"""CLI e2e coverage for `sqb rename model:` and `sqb mv` on a real DuckDB project.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + DryRunE2ETestCase, + ModelMigrationE2ETestCase, + ModelRefactorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + CUSTOMER_DECLARATIONS, + ORDER_SHAPE, + fragments_present, + load_raw_orders, + order_history_copy, + order_history_files, + order_ids, + project_text, + relation_type, + remove_files, + sqb, + sqb_json, + strip_migration_fingerprints, + write_orders_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ModelRefactorE2ETestCase( + description="rename keeps the folder and rewrites refs and fixtures", + command=("rename", "model:stg_orders", "stg_order_lines"), + expected_removed="models/staging/stg_orders.sql", + expected_file="models/staging/stg_order_lines.sql", + expected_fragments={ + "models/marts/fact_orders.sql": ('__ref("stg_order_lines")',), + "tests/unit/test_fact_orders.sql": ("__ref__stg_order_lines AS (",), + }, + expected_relation="stg_order_lines", + ), + ModelRefactorE2ETestCase( + description="rename rewrites seed and source relationships audits", + command=("rename", "model:stg_orders", "stg_order_lines"), + expected_removed="models/staging/stg_orders.sql", + expected_file="models/staging/stg_order_lines.sql", + expected_fragments={ + "seeds/customers.yml": ('to: __ref("stg_order_lines")',), + "sources/customers.yml": ("to: '__ref(\"stg_order_lines\")'",), + "models/staging/stg_order_lines.sql": ("migrate_from stg_orders",), + }, + expected_relation="stg_order_lines", + extra_files=CUSTOMER_DECLARATIONS, + ), + ModelRefactorE2ETestCase( + description="rename rewrites reusable schema relationships audits", + command=("rename", "model:stg_orders", "stg_order_lines"), + expected_removed="models/staging/stg_orders.sql", + expected_file="models/staging/stg_order_lines.sql", + expected_fragments={ + "models/marts/_sqlbuild/_schemas/order_shape.sql": ( + '(to __ref("stg_order_lines"), field customer_id)', + ), + }, + expected_relation="stg_order_lines", + extra_files=ORDER_SHAPE, + ), + ModelRefactorE2ETestCase( + description="rename rewrites cursor_inputs keys of incremental consumers", + command=("rename", "model:stg_orders", "stg_order_lines"), + expected_removed="models/staging/stg_orders.sql", + expected_file="models/staging/stg_order_lines.sql", + expected_fragments={ + "models/marts/order_history.sql": ( + "stg_order_lines order_date,", + '__ref("stg_order_lines")', + ), + }, + expected_relation="order_history", + extra_files=order_history_files(materialized="incremental", udf=False), + ), + ModelRefactorE2ETestCase( + description="move to a new file renames the model after the file", + command=("mv", "model:fact_orders", "models/reporting/order_facts.sql"), + expected_removed="models/marts/fact_orders.sql", + expected_file="models/reporting/order_facts.sql", + expected_fragments={ + "models/marts/customer_totals.sql": ('__ref("order_facts")',), + "tests/unit/test_fact_orders.sql": ("__expected__order_facts AS (",), + }, + expected_relation="order_facts", + ), + ModelRefactorE2ETestCase( + description="move into a folder keeps the model name", + command=("mv", "model:customer_totals", "models/reporting/"), + expected_removed="models/marts/customer_totals.sql", + expected_file="models/reporting/customer_totals.sql", + expected_fragments={}, + expected_relation="customer_totals", + ), + ], + ids=lambda case: case.description, +) +def test_given_model_refactor_when_applied_then_project_tests_and_builds( + tmp_path: Path, test_case: ModelRefactorE2ETestCase +) -> None: + """Every reference and fixture follows the model, and the edited project really runs.""" + + project_dir: Path = write_orders_project(tmp_path=tmp_path, files=test_case.extra_files) + + result: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.command) + tested: subprocess.CompletedProcess[str] = sqb(project_dir, "test") + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert result.returncode == 0, result.stdout + result.stderr + assert "Compiled: ok" in result.stdout + assert not (project_dir / test_case.expected_removed).exists() + assert (project_dir / test_case.expected_file).is_file() + files: dict[str, str] = project_text(project_dir) + assert fragments_present(files=files, expected=test_case.expected_fragments), files + assert tested.returncode == 0, tested.stdout + tested.stderr + assert built.returncode == 0, built.stdout + built.stderr + assert relation_type(project_dir=project_dir, name=test_case.expected_relation) is not None + + +@pytest.mark.parametrize( + "test_case", + [ + DryRunE2ETestCase( + description="model rename", + command=("rename", "model:stg_orders", "stg_order_lines", "--dry-run"), + expected_paths=frozenset( + { + "models/staging/stg_order_lines.sql", + "models/marts/fact_orders.sql", + "tests/unit/test_fact_orders.sql", + } + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_dry_run_json_when_refactoring_then_nothing_is_written( + tmp_path: Path, test_case: DryRunE2ETestCase +) -> None: + """A verified dry run reports its edits as JSON on stdout and progress on stderr.""" + + project_dir: Path = write_orders_project(tmp_path=tmp_path) + before: dict[str, str] = project_text(project_dir) + + result: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.command, "--json") + code: int + payload: dict[str, Any] + code, payload = sqb_json(project_dir, *test_case.command) + + assert result.returncode == 0, result.stderr + assert "Compiling project" in result.stderr + assert code == 0 + assert payload["status"] == "dry_run" + assert payload["files_changed"] == 0 + assert payload["compile"]["ok"] is True + assert {item["path"] for item in payload["files"]} == test_case.expected_paths + assert project_text(project_dir) == before + + +@pytest.mark.parametrize( + "test_case", + [ + ModelMigrationE2ETestCase( + description="table", + materialized="table", + udf=False, + expected_declaration="migrate_from order_history", + expected_old_name_type="VIEW", + expected_order_ids=(3, 4), + ), + ModelMigrationE2ETestCase( + description="view calling a UDF", + materialized="view", + udf=True, + expected_declaration="migrate_from order_history", + expected_old_name_type="VIEW", + expected_order_ids=(3, 4), + ), + ModelMigrationE2ETestCase( + description="incremental with an unchanged definition", + materialized="incremental", + udf=False, + expected_declaration="migrate_from order_history", + expected_old_name_type="VIEW", + expected_order_ids=(1, 2, 3, 4), + ), + ModelMigrationE2ETestCase( + description="incremental that discovery would find ambiguous", + materialized="incremental", + udf=False, + expected_declaration="migrate_from order_history", + expected_old_name_type="VIEW", + expected_order_ids=(1, 2, 3, 4), + extra_files=order_history_copy(), + removed_files=("models/marts/order_history_copy.sql",), + ), + ModelMigrationE2ETestCase( + description="incremental built before rename matching", + materialized="incremental", + udf=False, + expected_declaration="migrate_from order_history", + expected_old_name_type="VIEW", + expected_order_ids=(1, 2, 3, 4), + stripped_fingerprints=("order_history",), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_model_when_renamed_then_build_migrates_its_relation( + tmp_path: Path, test_case: ModelMigrationE2ETestCase +) -> None: + """Every rename declares migrate_from, so history survives whatever discovery would do.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, + files={ + **order_history_files(materialized=test_case.materialized, udf=test_case.udf), + **test_case.extra_files, + }, + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + strip_migration_fingerprints(project_dir=project_dir, models=test_case.stripped_fingerprints) + + result: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "model:order_history", "order_ledger" + ) + remove_files(project_dir=project_dir, paths=test_case.removed_files) + header: str = (project_dir / "models/marts/order_ledger.sql").read_text(encoding="utf-8") + load_raw_orders(project_dir=project_dir, order_ids=(3, 4)) + second: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert first.returncode == 0, first.stdout + first.stderr + assert result.returncode == 0, result.stdout + result.stderr + assert header.count(test_case.expected_declaration) == 1, header + assert second.returncode == 0, second.stdout + second.stderr + assert relation_type(project_dir=project_dir, name="order_history") == ( + test_case.expected_old_name_type + ) + assert order_ids(project_dir=project_dir, name="order_ledger") == test_case.expected_order_ids diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py index bdf8cafd20..d047d909fb 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py @@ -98,15 +98,6 @@ def test_given_interrupted_rename_when_retrying_then_column_is_renamed_once_and_ expected_decision="conflict", expected_error="error[M110]", ), - BlockedColumnRenameTestCase( - description="source column never existed", - renamed_sql=orders_sql( - columns="amount AS revenue", - extra_config=migrate_columns("revenue (migrate_from gross_amount)"), - ), - expected_decision="source_missing", - expected_error="error[M109]", - ), BlockedColumnRenameTestCase( description="source column is still produced", renamed_sql=orders_sql( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 4469ccbccc..c0e7c0a8c9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -33,6 +33,23 @@ old_name_views = false """ ).lstrip() +DENY_MISSING_ORIGIN_PROJECT_TOML: str = dedent( + f""" + name = "orders_project" + adapter = "duckdb" + default_target = "main" + + [connection] + database = "{DATABASE_FILE}" + + [targets.main] + schema = "main" + missing_migration_origin = "deny" + + [migrations] + old_name_views = false + """ +).lstrip() TARGETS_PROJECT_TOML: str = dedent( f""" name = "orders_project" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py index e9aaf3069b..46b51b2328 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py @@ -153,6 +153,8 @@ def test_given_renamed_chain_with_full_replay_when_building_then_migrated_histor "├── customer_orders_enriched migrate main.orders_enriched -> " "main.customer_orders_enriched\n ├── transfer recreate (view)\n" " ├── discovery automatic\n" + " ├── hint matched by unchanged definition; sqb rename model: " + "rewrites references and declares migrate_from so the move does not depend on discovery\n" " └── old name main.orders_enriched\n" " └── left for janitor old_name_view false" ), diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py index e58feff3fe..f0eaac192f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -16,6 +16,7 @@ MigrationOutcomeTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + DENY_MISSING_ORIGIN_PROJECT_TOML, DESTINATION_MODEL, ORIGIN_MODEL, PROJECT_TOML, @@ -330,12 +331,13 @@ def test_given_destination_with_build_history_when_building_then_conflict_until_ "test_case", [ MigrationOutcomeTestCase( - description="missing origin without a recorded migration fails the build", + description="missing origin fails the build when the target denies it", expected_decisions=("origin_missing",), expected_output_fragment=( "'stg_customer_orders' (migrate_from main.retired_orders) and no recorded " - "migration into it was found; if the migration already happened elsewhere or " - "is no longer needed, remove migrate_from from the model header" + "migration into it was found, and this target sets missing_migration_origin = " + "deny; if the migration already happened elsewhere or is no longer needed, " + "remove migrate_from from the model header" ), ) ], @@ -347,6 +349,7 @@ def test_given_missing_origin_without_event_when_building_then_fails_before_buil write_project( project_dir=tmp_path, models={DESTINATION_MODEL: incremental_orders_sql(migrate_from="retired_orders")}, + project_toml=DENY_MISSING_ORIGIN_PROJECT_TOML, ) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) @@ -358,8 +361,9 @@ def test_given_missing_origin_without_event_when_building_then_fails_before_buil assert tuple(warning["severity"] for warning in plan["warnings"]) == ("error",) assert plan["warnings"][0]["message"] == ( "model 'stg_customer_orders': migrate_from origin main.retired_orders does not exist " - "and no recorded migration into it was found; if the migration already happened " - "elsewhere or is no longer needed, remove migrate_from from the model header" + "in this target and no recorded migration into it was found; the build stops " + "(missing_migration_origin = deny); if the migration already happened elsewhere or is " + "no longer needed, remove migrate_from" ) assert "origin missing main.retired_orders -> main.stg_customer_orders" in text.output assert result.exit_code == 1 diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 83ad6e4e03..7e9608c845 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -3,7 +3,7 @@ from pathlib import Path from sqlbuild.spec.contracts.models import AuthoredTimeTravelRetention -from sqlbuild.spec.contracts.types import RetentionDecreasePolicy +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, RetentionDecreasePolicy from sqlbuild.sql_values.types import CollectionRendering @@ -896,3 +896,10 @@ class ParseSeedAuditsYamlTestCase: contents: str expected_table_audits: tuple[str, ...] expected_column_audits: tuple[tuple[str, ...], ...] + + +@dataclass(frozen=True) +class MissingMigrationOriginConfigTestCase: + description: str + target_lines: tuple[str, ...] + expected_policy: MissingMigrationOriginPolicy diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index 9ab93dc182..d4e94e1f67 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -18,7 +18,7 @@ ProjectConfig, TargetConfig, ) -from sqlbuild.spec.contracts.types import RetentionDecreasePolicy +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, RetentionDecreasePolicy from sqlbuild.sql_values.types import CollectionRendering from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ColumnContractModeConfigErrorTestCase, @@ -42,6 +42,7 @@ LoadTargetRetentionPoliciesTestCase, MicrobatchLimitConfigErrorTestCase, MicrobatchLimitConfigTestCase, + MissingMigrationOriginConfigTestCase, StartCursorConfigTestCase, TargetExecutionLimitsConfigTestCase, ) @@ -559,6 +560,43 @@ def test_given_target_retention_policies_when_loading_project_then_they_are_type assert target.time_travel_retention_decrease is test_case.expected_decrease_policy +@pytest.mark.parametrize( + "test_case", + [ + MissingMigrationOriginConfigTestCase( + description="missing migration origins are allowed by default", + target_lines=(), + expected_policy=MissingMigrationOriginPolicy.ALLOW, + ), + MissingMigrationOriginConfigTestCase( + description="deny is typed", + target_lines=('missing_migration_origin = "deny"',), + expected_policy=MissingMigrationOriginPolicy.DENY, + ), + MissingMigrationOriginConfigTestCase( + description="require_confirmation is typed", + target_lines=('missing_migration_origin = "require_confirmation"',), + expected_policy=MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION, + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_migration_origin_policy_when_loading_project_then_it_is_typed( + test_case: MissingMigrationOriginConfigTestCase, + tmp_path: Path, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + "\n".join( + ('name = "demo"', 'adapter = "duckdb"', "[targets.prod]", *test_case.target_lines) + ), + encoding="utf-8", + ) + + target: TargetConfig = load_project_config(project_dir=tmp_path).targets["prod"] + + assert target.missing_migration_origin is test_case.expected_policy + + @pytest.mark.parametrize( "test_case", [ @@ -577,6 +615,11 @@ def test_given_target_retention_policies_when_loading_project_then_they_are_type target_lines=('time_travel_retention_decrease = "sometimes"',), expected_error_fragment="'deny', 'require_confirmation', or 'allow'", ), + LoadTargetRetentionPoliciesErrorTestCase( + description="missing migration origin policy must be a known value", + target_lines=('missing_migration_origin = "sometimes"',), + expected_error_fragment="targets.prod.missing_migration_origin must be", + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_test_types.py index e69de29bb2..5563adc22d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_test_types.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy + + +@dataclass(frozen=True) +class MissingOriginBlockTestCase: + description: str + origin_tracked: bool + policy: MissingMigrationOriginPolicy + expected_blocks: bool diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index 583a7ca25a..cfec0adb75 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -92,11 +92,11 @@ ), ), MigrationDecisionStyleTestCase( - description="origin missing is bold red", + description="origin missing allowed by the default policy is bold yellow", decision=MigrationDecision.ORIGIN_MISSING, compatibility=MigrationCompatibility.NOT_CHECKED, findings=(), - expected_fragments=(f"{_BOLD_RED}origin missing{_RESET}",), + expected_fragments=(f"{_BOLD_YELLOW}origin missing{_RESET}",), ), MigrationDecisionStyleTestCase( description="renamed table is listed as a bold blue migration that rebuilds it", @@ -138,7 +138,9 @@ def test_given_migration_decision_when_formatting_then_applies_colour_roles( "└── daily_revenue migrate prod.revenue -> prod.daily_revenue\n" " ├── compatibility compatible\n" " ├── transfer physical copy, promote by transactional rename\n" - " └── discovery automatic" + " ├── discovery automatic\n" + " └── hint matched by unchanged definition; sqb rename model: " + "rewrites references and declares migrate_from so the move does not depend on discovery" ), ), ), diff --git a/tests/unit/src/sqlbuild/compiler/planner/test_models.py b/tests/unit/src/sqlbuild/compiler/planner/test_models.py index 8b13789179..847e0b6039 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/test_models.py +++ b/tests/unit/src/sqlbuild/compiler/planner/test_models.py @@ -1 +1,62 @@ +"""Tests for planner model decisions.""" +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from sqlbuild.compiler.migrations.types import MigrationDecision +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy +from tests.unit.src.sqlbuild.compiler.planner._test_types import MissingOriginBlockTestCase +from tests.unit.src.sqlbuild.compiler.planner.main.pre_build.helpers import migration_entry + + +@pytest.mark.parametrize( + "test_case", + [ + MissingOriginBlockTestCase( + description="never-built origin follows allow", + origin_tracked=False, + policy=MissingMigrationOriginPolicy.ALLOW, + expected_blocks=False, + ), + MissingOriginBlockTestCase( + description="never-built origin follows require_confirmation", + origin_tracked=False, + policy=MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION, + expected_blocks=False, + ), + MissingOriginBlockTestCase( + description="never-built origin follows deny", + origin_tracked=False, + policy=MissingMigrationOriginPolicy.DENY, + expected_blocks=True, + ), + MissingOriginBlockTestCase( + description="origin built in this target blocks even under allow", + origin_tracked=True, + policy=MissingMigrationOriginPolicy.ALLOW, + expected_blocks=True, + ), + MissingOriginBlockTestCase( + description="origin built in this target blocks under require_confirmation", + origin_tracked=True, + policy=MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION, + expected_blocks=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_when_deciding_then_history_overrides_policy( + test_case: MissingOriginBlockTestCase, +) -> None: + entry: ModelMigrationPlanEntry = replace( + migration_entry(decision=MigrationDecision.ORIGIN_MISSING), + origin_tracked=test_case.origin_tracked, + missing_origin_policy=test_case.policy, + ) + + assert entry.blocks_build is test_case.expected_blocks + assert entry.origin_hidden is test_case.origin_tracked diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/__init__.py b/tests/unit/src/sqlbuild/compiler/refactoring/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/__init__.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py new file mode 100644 index 0000000000..196c084397 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py @@ -0,0 +1,62 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ColumnEditTestCase: + description: str + sql: str + cascade: bool + expected_sql: str + expected_manual: str = "" + expected_passes_through: bool = False + + +@dataclass(frozen=True) +class HeaderEditTestCase: + description: str + contents: str + expected_contents: str + + +@dataclass(frozen=True) +class IdentifierSitesTestCase: + description: str + text: str + expected_offsets: tuple[int, ...] + + +@dataclass(frozen=True) +class ApplyEditsTestCase: + description: str + text: str + spans: tuple[tuple[int, int, str], ...] + expected_text: str + + +@dataclass(frozen=True) +class ApplyEditsErrorTestCase: + description: str + text: str + spans: tuple[tuple[int, int, str], ...] + expected_error: type[Exception] + + +@dataclass(frozen=True) +class CommitChangesTestCase: + description: str + files: dict[str, str] + current_files: dict[str, str] + locked_directory: str + spans: tuple[tuple[int, int, str], ...] + edited_path: str + moved_from: str + moved_to: str + expected_error: type[Exception] + expected_files: dict[str, str] + + +@dataclass(frozen=True) +class DeclarationEditTestCase: + description: str + contents: str + expected_contents: str diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py new file mode 100644 index 0000000000..cfe5233aff --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py @@ -0,0 +1,117 @@ +"""Shared builders for refactoring helper tests.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.refactoring._helpers.columns.column_references import ( + analyze_column, + consumer_edits, +) +from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( + add_column_entry_edit, + column_config_edits, + column_entry_edits, + header_tokens, +) +from sqlbuild.compiler.refactoring._helpers.text.sql_sites import analysis_sql +from sqlbuild.compiler.refactoring._helpers.text.text_edits import text_edit +from sqlbuild.compiler.refactoring.models import ( + AnalysisSql, + BodyContext, + BodyEdits, + ColumnFacts, + ColumnQuery, + HeaderToken, + ProjectSqlFile, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import EditKind, SqlFileRole + +DIALECT: str = "duckdb" +COLUMNS: dict[tuple[str, str], tuple[str, ...]] = { + ("ref", "fact_orders"): ("order_id", "customer_id", "amount"), + ("ref", "customers"): ("customer_id", "amount"), +} + + +def plan_consumer(*, sql: str, cascade: bool) -> BodyEdits: + """Plan renaming fact_orders.amount to revenue in one consumer query.""" + + analysis: AnalysisSql = analysis_sql(text=sql, dialect=DIALECT) + facts: ColumnFacts = analyze_column( + analysis=analysis, + dialect=DIALECT, + columns=COLUMNS, + query=ColumnQuery(column="amount", target_tables=analysis.tables[("ref", "fact_orders")]), + ) + return consumer_edits( + facts=facts, + context=BodyContext( + path="models/consumer.sql", + contents=sql, + map_span=lambda start, end: (start, end), + locate=lambda offset: offset, + fallback_offset=0, + ), + old="amount", + new="revenue", + cascade_root=cascade, + root_stars_pass=cascade, + ) + + +def migrated_column_edits(contents: str) -> tuple[TextEdit, ...]: + """Plan the header edits that rename amount to revenue with a column migration.""" + + tokens: tuple[HeaderToken, ...] = header_tokens(contents=contents) or () + added: tuple[TextEdit, ...] = tuple( + filter( + None, + (add_column_entry_edit(contents=contents, tokens=tokens, new="revenue", old="amount"),), + ) + ) + return ( + *column_config_edits(contents=contents, tokens=tokens, old="amount", new="revenue"), + *( + column_entry_edits( + contents=contents, tokens=tokens, old="amount", new="revenue", migrate=True + ) + or added + ), + ) + + +def span_edits(*, text: str, spans: tuple[tuple[int, int, str], ...]) -> tuple[TextEdit, ...]: + """Build column edits for (start, end, replacement) spans.""" + + return tuple( + text_edit(text=text, start=start, end=end, replacement=replacement, kind=EditKind.COLUMN) + for start, end, replacement in spans + ) + + +def write_tree(*, root: Path, files: dict[str, str]) -> None: + """Write project-relative files under a root.""" + + relative: str + contents: str + for relative, contents in files.items(): + _ = (root / relative).write_text(contents, encoding="utf-8") + + +def read_tree(*, root: Path) -> dict[str, str]: + """Read every file under a root, keyed by its relative path.""" + + return { + path.relative_to(root).as_posix(): path.read_text(encoding="utf-8") + for path in sorted(root.rglob("*.sql")) + } + + +def yaml_file(contents: str) -> ProjectSqlFile: + """Wrap YAML text as one project declaration file.""" + + return ProjectSqlFile( + relative_path="seeds/customers.yml", contents=contents, role=SqlFileRole.YAML + ) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py new file mode 100644 index 0000000000..6baa3aebc1 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py @@ -0,0 +1,104 @@ +"""Tests for column-rename edit planning inside one SQL body.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring.models import BodyEdits +from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ColumnEditTestCase +from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import plan_consumer + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnEditTestCase( + description="bare projection keeps its output name", + sql='SELECT o.amount FROM __ref("fact_orders") AS o', + cascade=False, + expected_sql='SELECT o.revenue AS amount FROM __ref("fact_orders") AS o', + ), + ColumnEditTestCase( + description="cascade renames a bare projection's output", + sql='SELECT o.amount FROM __ref("fact_orders") AS o', + cascade=True, + expected_sql='SELECT o.revenue FROM __ref("fact_orders") AS o', + expected_passes_through=True, + ), + ColumnEditTestCase( + description="aliased projection keeps its alias", + sql='SELECT o.amount AS total FROM __ref("fact_orders") AS o', + cascade=True, + expected_sql='SELECT o.revenue AS total FROM __ref("fact_orders") AS o', + ), + ColumnEditTestCase( + description="filter reference is renamed", + sql='SELECT order_id FROM __ref("fact_orders") WHERE amount > 0', + cascade=False, + expected_sql='SELECT order_id FROM __ref("fact_orders") WHERE revenue > 0', + ), + ColumnEditTestCase( + description="same-named column of another relation is untouched", + sql=( + 'SELECT c.amount FROM __ref("fact_orders") AS o ' + 'JOIN __ref("customers") AS c ON o.customer_id = c.customer_id' + ), + cascade=False, + expected_sql=( + 'SELECT c.amount FROM __ref("fact_orders") AS o ' + 'JOIN __ref("customers") AS c ON o.customer_id = c.customer_id' + ), + ), + ColumnEditTestCase( + description="star through a CTE is followed", + sql='WITH c AS (SELECT * FROM __ref("fact_orders")) SELECT c.amount FROM c', + cascade=False, + expected_sql=( + 'WITH c AS (SELECT * FROM __ref("fact_orders")) SELECT c.revenue AS amount FROM c' + ), + ), + ColumnEditTestCase( + description="root star needs cascade", + sql='SELECT * FROM __ref("fact_orders")', + cascade=False, + expected_sql='SELECT * FROM __ref("fact_orders")', + expected_manual="rerun with --cascade", + ), + ColumnEditTestCase( + description="root star passes through under cascade", + sql='SELECT * FROM __ref("fact_orders")', + cascade=True, + expected_sql='SELECT * FROM __ref("fact_orders")', + expected_passes_through=True, + ), + ColumnEditTestCase( + description="USING join is left for the author", + sql=( + 'SELECT o.order_id FROM __ref("fact_orders") AS o ' + 'JOIN __ref("customers") AS c USING (amount)' + ), + cascade=False, + expected_sql=( + 'SELECT o.order_id FROM __ref("fact_orders") AS o ' + 'JOIN __ref("customers") AS c USING (amount)' + ), + expected_manual="USING or NATURAL join", + ), + ], + ids=lambda case: case.description, +) +def test_given_consumer_sql_when_planning_column_rename_then_edits_match( + test_case: ColumnEditTestCase, +) -> None: + result: BodyEdits = plan_consumer(sql=test_case.sql, cascade=test_case.cascade) + + assert apply_text_edits(text=test_case.sql, edits=result.edits) == test_case.expected_sql + assert result.passes_through == test_case.expected_passes_through + reasons: tuple[str, ...] = tuple(location.reason for location in result.manual) + assert bool(reasons) == bool(test_case.expected_manual), reasons + assert all(test_case.expected_manual in reason for reason in reasons), reasons + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_declaration_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_declaration_edits.py new file mode 100644 index 0000000000..5f922889e0 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_declaration_edits.py @@ -0,0 +1,232 @@ +"""Tests for model and column reference rewriting in YAML and declaration headers.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( + consumer_column_header_edits, + model_name_header_edits, +) +from sqlbuild.compiler.refactoring._helpers.text.schema_edits import ( + schema_column_edits, + schema_model_name_edits, +) +from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring._helpers.text.yaml_edits import ( + yaml_column_edits, + yaml_model_edits, +) +from sqlbuild.compiler.refactoring.models import TextEdit +from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( + DeclarationEditTestCase, +) +from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import yaml_file + +_BODY: str = "SELECT order_id FROM orders\n" + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="relationships target written as a reference", + contents=( + "seeds:\n - name: customers\n columns:\n - name: customer_id\n" + " audits:\n - relationships:\n" + ' to: __ref("fact_orders")\n field: customer_id\n' + ), + expected_contents=( + "seeds:\n - name: customers\n columns:\n - name: customer_id\n" + " audits:\n - relationships:\n" + ' to: __ref("order_facts")\n field: customer_id\n' + ), + ), + DeclarationEditTestCase( + description="bare relationships target in a flow mapping", + contents="audits:\n - relationships: {to: fact_orders, field: customer_id}\n", + expected_contents="audits:\n - relationships: {to: order_facts, field: customer_id}\n", + ), + DeclarationEditTestCase( + description="reference inside an escaped double-quoted expression", + contents='audits:\n - expression_is_true: {expression: "id IN (SELECT id FROM ' + '__ref(\\"fact_orders\\"))"}\n', + expected_contents='audits:\n - expression_is_true: {expression: "id IN (SELECT id ' + 'FROM __ref(\\"order_facts\\"))"}\n', + ), + DeclarationEditTestCase( + description="similar names and descriptions are left alone", + contents=( + "sources:\n - name: raw\n description: fact_orders feed\n" + " audits:\n - relationships: {to: fact_orders_daily, field: id}\n" + ), + expected_contents=( + "sources:\n - name: raw\n description: fact_orders feed\n" + " audits:\n - relationships: {to: fact_orders_daily, field: id}\n" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_yaml_when_renaming_model_then_references_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[tuple[str, TextEdit], ...] = yaml_model_edits( + files=(yaml_file(test_case.contents),), old="fact_orders", new="order_facts" + ) + + assert ( + apply_text_edits(text=test_case.contents, edits=tuple(edit for _, edit in edits)) + == test_case.expected_contents + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="field of a relationship to the model", + contents="audits:\n - relationships: {to: '__ref(\"fact_orders\")', field: amount}\n", + expected_contents=( + "audits:\n - relationships: {to: '__ref(\"fact_orders\")', field: revenue}\n" + ), + ), + DeclarationEditTestCase( + description="field of a relationship to another model", + contents="audits:\n - relationships: {to: '__ref(\"customers\")', field: amount}\n", + expected_contents=( + "audits:\n - relationships: {to: '__ref(\"customers\")', field: amount}\n" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_yaml_when_renaming_column_then_relationship_fields_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[tuple[str, TextEdit], ...] = yaml_column_edits( + files=(yaml_file(test_case.contents),), model="fact_orders", old="amount", new="revenue" + ) + + assert ( + apply_text_edits(text=test_case.contents, edits=tuple(edit for _, edit in edits)) + == test_case.expected_contents + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="MODEL header quoted SQL and bare relationships target", + contents=( + 'MODEL (\n audits [expression_is_true (expression "id IN (SELECT id FROM ' + "__ref('fact_orders'))\")],\n columns (\n id (audits [relationships " + f"(to fact_orders, field id)]),\n ),\n);\n{_BODY}" + ), + expected_contents=( + 'MODEL (\n audits [expression_is_true (expression "id IN (SELECT id FROM ' + "__ref('order_facts'))\")],\n columns (\n id (audits [relationships " + f"(to order_facts, field id)]),\n ),\n);\n{_BODY}" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_model_header_when_renaming_model_then_quoted_and_bare_references_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = model_name_header_edits( + contents=test_case.contents, old="fact_orders", new="order_facts" + ) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="field of a referenced relationship target", + contents=( + "MODEL (\n columns (\n id (audits [relationships " + f'(to __ref("fact_orders"), field amount)]),\n ),\n);\n{_BODY}' + ), + expected_contents=( + "MODEL (\n columns (\n id (audits [relationships " + f'(to __ref("fact_orders"), field revenue)]),\n ),\n);\n{_BODY}' + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_model_header_when_renaming_column_then_relationship_fields_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = consumer_column_header_edits( + contents=test_case.contents, upstream="fact_orders", old="amount", new="revenue" + ) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="bare target and quoted SQL in every SCHEMA of a file", + contents=( + "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " + "(to fact_orders, field id)]),\n ),\n);\n\n" + "SCHEMA (\n name order_totals,\n audits [expression_is_true (expression " + "\"total > (SELECT 0 FROM __ref('fact_orders') LIMIT 1)\")],\n);\n" + ), + expected_contents=( + "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " + "(to order_facts, field id)]),\n ),\n);\n\n" + "SCHEMA (\n name order_totals,\n audits [expression_is_true (expression " + "\"total > (SELECT 0 FROM __ref('order_facts') LIMIT 1)\")],\n);\n" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_schema_file_when_renaming_model_then_references_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = schema_model_name_edits( + contents=test_case.contents, old="fact_orders", new="order_facts" + ) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="field of a relationship to the model", + contents=( + "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " + '(to __ref("fact_orders"), field amount)]),\n ),\n);\n' + ), + expected_contents=( + "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " + '(to __ref("fact_orders"), field revenue)]),\n ),\n);\n' + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_schema_file_when_renaming_column_then_relationship_fields_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = schema_column_edits( + contents=test_case.contents, upstream="fact_orders", old="amount", new="revenue" + ) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py new file mode 100644 index 0000000000..78159601c5 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py @@ -0,0 +1,130 @@ +"""Tests for token-exact MODEL header edits.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( + insert_header_entry_edit, + model_name_header_edits, +) +from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring.models import TextEdit +from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import HeaderEditTestCase +from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import migrated_column_edits + +_BODY: str = "SELECT order_id, amount FROM orders\n" + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderEditTestCase( + description="declared column gains migrate_from before its metadata", + contents=f"MODEL (\n columns (\n amount (nullable false),\n ),\n);\n{_BODY}", + expected_contents=( + "MODEL (\n columns (\n revenue (migrate_from amount, nullable false),\n" + f" ),\n);\n{_BODY}" + ), + ), + HeaderEditTestCase( + description="bare declared column gains a migrate_from block", + contents=f"MODEL (\n columns (amount),\n);\n{_BODY}", + expected_contents=f"MODEL (\n columns (revenue (migrate_from amount)),\n);\n{_BODY}", + ), + HeaderEditTestCase( + description="undeclared column gets a new columns entry", + contents=f"MODEL (\n materialized incremental,\n);\n{_BODY}", + expected_contents=( + "MODEL (\n columns (revenue (migrate_from amount)),\n" + f" materialized incremental,\n);\n{_BODY}" + ), + ), + HeaderEditTestCase( + description="column-valued config follows the column", + contents=f"MODEL (\n unique_key [order_id, amount],\n);\n{_BODY}", + expected_contents=( + "MODEL (\n columns (revenue (migrate_from amount)),\n" + f" unique_key [order_id, revenue],\n);\n{_BODY}" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_header_when_renaming_migrated_column_then_header_is_rewritten( + test_case: HeaderEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = migrated_column_edits(test_case.contents) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderEditTestCase( + description="cursor_inputs keys and relationships targets follow the model", + contents=( + "MODEL (\n cursor_inputs (\n fact_orders ordered_at,\n ),\n" + " columns (\n order_id (audits [relationships (to fact_orders, " + "field order_id)]),\n ),\n);\n" + f"{_BODY}" + ), + expected_contents=( + "MODEL (\n cursor_inputs (\n order_facts ordered_at,\n ),\n" + " columns (\n order_id (audits [relationships (to order_facts, " + "field order_id)]),\n ),\n);\n" + f"{_BODY}" + ), + ), + HeaderEditTestCase( + description="descriptions naming the model are left alone", + contents=f'MODEL (\n description "fact_orders summary",\n);\n{_BODY}', + expected_contents=f'MODEL (\n description "fact_orders summary",\n);\n{_BODY}', + ), + ], + ids=lambda case: case.description, +) +def test_given_header_when_renaming_model_then_references_follow( + test_case: HeaderEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = model_name_header_edits( + contents=test_case.contents, old="fact_orders", new="order_facts" + ) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderEditTestCase( + description="multi-line header gets its own line", + contents=f"MODEL (\n materialized table,\n);\n{_BODY}", + expected_contents=( + f"MODEL (\n migrate_from fact_orders,\n materialized table,\n);\n{_BODY}" + ), + ), + HeaderEditTestCase( + description="empty header is expanded", + contents=f"MODEL ();\n{_BODY}", + expected_contents=f"MODEL (\n migrate_from fact_orders,\n);\n{_BODY}", + ), + ], + ids=lambda case: case.description, +) +def test_given_header_when_adding_model_migration_then_entry_is_inserted( + test_case: HeaderEditTestCase, +) -> None: + edit: TextEdit | None = insert_header_entry_edit( + contents=test_case.contents, + entry="migrate_from fact_orders", + display="migrate_from fact_orders", + ) + + assert edit is not None + assert apply_text_edits(text=test_case.contents, edits=(edit,)) == test_case.expected_contents + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py new file mode 100644 index 0000000000..d68fad0735 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py @@ -0,0 +1,95 @@ +"""Tests for authored-text edit application and identifier matching.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.refactoring._helpers.text.text_edits import ( + apply_text_edits, + identifier_sites, +) +from sqlbuild.compiler.refactoring.exceptions import RefactorEditError +from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( + ApplyEditsErrorTestCase, + ApplyEditsTestCase, + IdentifierSitesTestCase, +) +from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import span_edits + + +@pytest.mark.parametrize( + "test_case", + [ + ApplyEditsTestCase( + description="identical duplicate edits apply once", + text="SELECT amount FROM orders", + spans=((7, 13, "revenue"), (7, 13, "revenue")), + expected_text="SELECT revenue FROM orders", + ), + ApplyEditsTestCase( + description="insertions and replacements apply back to front", + text="SELECT amount FROM orders", + spans=((7, 13, "revenue"), (13, 13, " AS amount")), + expected_text="SELECT revenue AS amount FROM orders", + ), + ], + ids=lambda case: case.description, +) +def test_given_edits_when_applying_then_text_is_rewritten(test_case: ApplyEditsTestCase) -> None: + assert ( + apply_text_edits( + text=test_case.text, edits=span_edits(text=test_case.text, spans=test_case.spans) + ) + == test_case.expected_text + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ApplyEditsErrorTestCase( + description="overlapping replacements", + text="SELECT amount FROM orders", + spans=((7, 13, "revenue"), (10, 18, "x")), + expected_error=RefactorEditError, + ) + ], + ids=lambda case: case.description, +) +def test_given_overlapping_edits_when_applying_then_refuses( + test_case: ApplyEditsErrorTestCase, +) -> None: + with pytest.raises(test_case.expected_error): + _ = apply_text_edits( + text=test_case.text, edits=span_edits(text=test_case.text, spans=test_case.spans) + ) + + +@pytest.mark.parametrize( + "test_case", + [ + IdentifierSitesTestCase( + description="matches whole unquoted identifiers only", + text="__ref__orders AS (SELECT 1), __ref__orders_daily AS (SELECT 2)", + expected_offsets=(0,), + ), + IdentifierSitesTestCase( + description="skips comments and string literals", + text="-- __ref__orders\nSELECT '__ref__orders', __ref__orders", + expected_offsets=(41,), + ), + ], + ids=lambda case: case.description, +) +def test_given_text_when_finding_identifiers_then_code_sites_match( + test_case: IdentifierSitesTestCase, +) -> None: + sites: tuple[tuple[int, int, str], ...] = identifier_sites( + text=test_case.text, names=frozenset({"__ref__orders"}) + ) + + assert tuple(start for start, _, _ in sites) == test_case.expected_offsets + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py new file mode 100644 index 0000000000..c21491c1f0 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py @@ -0,0 +1,76 @@ +"""Tests for committing verified refactoring edits into a project.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.refactoring._helpers.project.workspace import commit_changes +from sqlbuild.compiler.refactoring.exceptions import RefactorWriteError +from sqlbuild.compiler.refactoring.models import FileChange +from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( + CommitChangesTestCase, +) +from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import ( + read_tree, + span_edits, + write_tree, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CommitChangesTestCase( + description="a failed write restores every file", + files={"a.sql": "SELECT amount", "m.sql": "SELECT 1"}, + current_files={}, + locked_directory="locked", + spans=((7, 13, "revenue"),), + edited_path="a.sql", + moved_from="m.sql", + moved_to="locked/m.sql", + expected_error=PermissionError, + expected_files={"a.sql": "SELECT amount", "m.sql": "SELECT 1"}, + ), + CommitChangesTestCase( + description="a file edited since planning stops the write", + files={"a.sql": "SELECT amount", "m.sql": "SELECT 1"}, + current_files={"a.sql": "SELECT amount, 2"}, + locked_directory="open", + spans=((7, 13, "revenue"),), + edited_path="a.sql", + moved_from="m.sql", + moved_to="open/m.sql", + expected_error=RefactorWriteError, + expected_files={"a.sql": "SELECT amount, 2", "m.sql": "SELECT 1"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_commit_when_writing_then_project_is_unchanged( + tmp_path: Path, test_case: CommitChangesTestCase +) -> None: + write_tree(root=tmp_path, files={**test_case.files, **test_case.current_files}) + (tmp_path / test_case.locked_directory).mkdir() + (tmp_path / "locked").mkdir(exist_ok=True) + (tmp_path / "locked").chmod(0o500) + changes: tuple[FileChange, ...] = ( + FileChange( + path=test_case.edited_path, + original_path=test_case.edited_path, + edits=span_edits(text=test_case.files[test_case.edited_path], spans=test_case.spans), + ), + FileChange(path=test_case.moved_to, original_path=test_case.moved_from), + ) + + with pytest.raises(test_case.expected_error): + _ = commit_changes(project_dir=tmp_path, originals=test_case.files, changes=changes) + (tmp_path / "locked").chmod(0o700) + + assert read_tree(root=tmp_path) == test_case.expected_files + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py index 97b6caedeb..785f725133 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py @@ -221,3 +221,11 @@ class TolerantCategoryCase: files: dict[str, str] expected_kind: ResourceKind | DeclarationKind expected_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class RelocationCase: + description: str + consumer_paths: tuple[str, ...] + destination: str + expected_paths: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py index 0ef42d0d8b..5e62b7decd 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py @@ -336,3 +336,49 @@ def report_scope_lookup(*, extra_globals: int = 0) -> ScopeLookup: completeness=ScopeCompleteness(), ) return build_scope_lookup(index=index) + + +_RELOCATION_ROOT: OwnershipRoot = OwnershipRoot("models", resource_kind=ResourceKind.MODEL) +_CENTS_MACRO: DeclarationIdentity = DeclarationIdentity(DeclarationKind.MACRO, "to_cents") +_ORDER_SHAPE: DeclarationIdentity = DeclarationIdentity(DeclarationKind.SCHEMA, "order_shape") + + +def relocation_index(consumer_paths: tuple[str, ...]) -> ScopeIndex: + """Return models sharing a staging schema that uses a staging macro.""" + + resources: tuple[ResourceRecord, ...] = tuple( + ResourceRecord( + identity=ResourceIdentity(ResourceKind.MODEL, f"orders_{position}"), + path=path, + ownership_root=_RELOCATION_ROOT, + ) + for position, path in enumerate(consumer_paths) + ) + return ScopeIndex( + ownership_roots=(_RELOCATION_ROOT,), + resources=resources, + declarations=( + DeclarationRecord( + identity=_ORDER_SHAPE, + path="models/staging/_sqlbuild/_schemas/order_shape.sql", + line=1, + column=1, + scope=ScopeKind.LOCAL, + ownership_root=_RELOCATION_ROOT, + owning_path="models/staging", + ), + DeclarationRecord( + identity=_CENTS_MACRO, + path="models/staging/_sqlbuild/_macros/cents.py", + line=1, + column=1, + scope=ScopeKind.LOCAL, + ownership_root=_RELOCATION_ROOT, + owning_path="models/staging", + ), + ), + usages=( + *(UsageRecord(consumer=item.identity, declaration=_ORDER_SHAPE) for item in resources), + UsageRecord(consumer=_ORDER_SHAPE, declaration=_CENTS_MACRO), + ), + ) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py new file mode 100644 index 0000000000..8f76f0ce39 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py @@ -0,0 +1,62 @@ +"""Behavior tests for relocating declarations when a resource moves.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.scopes.main.relocate_declarations_for_move import ( + relocate_declarations_for_move, +) +from sqlbuild.compiler.scopes.models import ( + DeclarationRecord, + ResourceIdentity, +) +from sqlbuild.compiler.scopes.types import ResourceKind +from tests.unit.src.sqlbuild.compiler.scopes._test_types import RelocationCase +from tests.unit.src.sqlbuild.compiler.scopes.helpers import relocation_index + + +@pytest.mark.parametrize( + "test_case", + ( + RelocationCase( + description="sole consumer takes the schema and the macro it uses along", + consumer_paths=("models/staging/orders.sql",), + destination="models/marts/orders.sql", + expected_paths=( + "models/marts/_sqlbuild/_schemas/order_shape.sql", + "models/marts/_sqlbuild/_macros/cents.py", + ), + ), + RelocationCase( + description="remaining consumers lift the schema while its macro stays beside it", + consumer_paths=("models/staging/orders.sql", "models/staging/refunds.sql"), + destination="models/staging/eu/orders.sql", + expected_paths=("models/staging/_sqlbuild/schemas/order_shape.sql",), + ), + RelocationCase( + description="consumers spanning the models root use top-level roles", + consumer_paths=("models/staging/orders.sql", "models/staging/refunds.sql"), + destination="models/marts/orders.sql", + expected_paths=("schemas/order_shape.sql", "macros/cents.py"), + ), + RelocationCase( + description="move within the owner changes nothing", + consumer_paths=("models/staging/orders.sql",), + destination="models/staging/orders_v2.sql", + expected_paths=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_resource_move_when_relocating_then_declarations_follow_placement( + test_case: RelocationCase, +) -> None: + relocated: tuple[DeclarationRecord, ...] | None = relocate_declarations_for_move( + index=relocation_index(test_case.consumer_paths), + resource=ResourceIdentity(ResourceKind.MODEL, "orders_0"), + destination=test_case.destination, + ) + + assert relocated is not None + assert tuple(item.path for item in relocated) == test_case.expected_paths diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 5a4971c5c2..8561369fad 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -22,6 +22,7 @@ class JanitorPlanTestCase: tracked_relations: tuple[tuple[str | None, str | None, str], ...] = field(default_factory=tuple) exclude_patterns: tuple[str, ...] = field(default_factory=tuple) protected_relation_keys: frozenset[JanitorRelationKey] = frozenset() + migrate_from: str | None = None expected_candidate_names: tuple[str, ...] = field(default_factory=tuple) expected_direct_state_table_names: tuple[str, ...] = field(default_factory=tuple) expected_skipped_relation_reasons: tuple[str, ...] = field(default_factory=tuple) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index aa0eb7cfbf..ae6494a16c 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -540,6 +540,7 @@ def build_project( source_database: str | None = None, destination_database: str | None = None, destination_schema: str = "analytics", + migrate_from: str | None = None, ) -> CompiledProject: source: CompiledSource = CompiledSource( key=CompiledObjectKey( @@ -569,7 +570,7 @@ def build_project( name="orders", relative_path=Path("models/orders.sql"), query_sql="select 1", - config=CompileModelConfig(), + config=CompileModelConfig(values={"migrate_from": migrate_from}), destination=CompiledRelationLocation( database=destination_database, schema=destination_schema, diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index c50974b1d2..8deb2b0c05 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -164,6 +164,29 @@ expected_candidate_names=(), expected_skipped_relation_reasons=("relation is not tracked by SQLBuild",), ), + JanitorPlanTestCase( + description="origin of a pending migration is kept", + relation_infos=(relation_info("old_orders", created_at=OLD_TIME),), + migrate_from="old_orders", + expected_candidate_names=(), + expected_skipped_relation_reasons=("pending migration origin for orders",), + ), + JanitorPlanTestCase( + description="qualified origin of a pending migration is kept", + relation_infos=(relation_info("OLD_ORDERS", created_at=OLD_TIME),), + migrate_from="analytics.old_orders", + expected_candidate_names=(), + expected_skipped_relation_reasons=("pending migration origin for orders",), + ), + JanitorPlanTestCase( + description="origin is cleaned normally once the destination exists", + relation_infos=( + relation_info("orders", created_at=OLD_TIME), + relation_info("old_orders", created_at=OLD_TIME), + ), + migrate_from="old_orders", + expected_candidate_names=("old_orders",), + ), JanitorPlanTestCase( description="existing direct state tables are eligible for history pruning", relation_infos=( @@ -204,7 +227,9 @@ def test_given_project_and_warehouse_when_building_janitor_plan_then_returns_exp ) plan: JanitorPlan = build_janitor_plan( - project=build_project(source_schema=test_case.source_schema), + project=build_project( + source_schema=test_case.source_schema, migrate_from=test_case.migrate_from + ), adapter=adapter, connection=object(), retention_days=test_case.retention_days, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py index ea16b93e94..bd195c7ca4 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py @@ -9,6 +9,7 @@ LocalConfig, ProjectConfig, ) +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy from sqlbuild.sql_values.types import CollectionRendering @@ -55,3 +56,12 @@ class InvalidLoaderDestinationTestCase: description: str destination: str expected_error_fragment: str + + +@dataclass(frozen=True) +class MissingOriginPolicyResolutionTestCase: + description: str + project_config: ProjectConfig + local_config: LocalConfig + target_name: str + expected_policy: MissingMigrationOriginPolicy diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py index 79bd7b2ac5..a01f23747d 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py @@ -11,8 +11,10 @@ ProjectConfig, TargetConfig, ) +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( ExecutionLimitsResolutionTestCase, + MissingOriginPolicyResolutionTestCase, TargetRetentionResolutionTestCase, ) @@ -134,3 +136,55 @@ def test_given_local_execution_limit_fields_when_resolving_then_each_field_overr ) assert target_config.execution_limits == test_case.expected_limits + + +@pytest.mark.parametrize( + "test_case", + [ + MissingOriginPolicyResolutionTestCase( + description="local target policy overrides the project target", + project_config=ProjectConfig( + name="shop", + adapter="duckdb", + targets={ + "dev": TargetConfig(missing_migration_origin=MissingMigrationOriginPolicy.DENY) + }, + ), + local_config=LocalConfig( + targets={ + "dev": LocalTargetConfig( + missing_migration_origin=MissingMigrationOriginPolicy.ALLOW + ) + } + ), + target_name="dev", + expected_policy=MissingMigrationOriginPolicy.ALLOW, + ), + MissingOriginPolicyResolutionTestCase( + description="project target policy applies without a local override", + project_config=ProjectConfig( + name="shop", + adapter="duckdb", + targets={ + "dev": TargetConfig( + missing_migration_origin=MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION + ) + }, + ), + local_config=LocalConfig(targets={"dev": LocalTargetConfig()}), + target_name="dev", + expected_policy=MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION, + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_policy_when_resolving_then_local_overrides_project( + test_case: MissingOriginPolicyResolutionTestCase, +) -> None: + target_config: TargetConfig = resolve_target_config( + project_config=test_case.project_config, + local_config=test_case.local_config, + target_name=test_case.target_name, + ) + + assert target_config.missing_migration_origin is test_case.expected_policy diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 55bdb6d92f..0c0868e469 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -59,6 +59,10 @@ A compatibility view is never archived as a stale relation. Once it expires, or Run the same SQLBuild version for `sqb janitor` as for builds. An older janitor does not know compatibility views and archives them as stale relations. +## Pending migration origins + +A relation that a project model still has to migrate from is never archived. While a model declares `migrate_from` and its own relation does not exist yet in the target, the janitor skips the origin and lists it as `pending migration origin for `. A bare `migrate_from` name is matched in each schema the project builds into; a qualified name only at that location. Once the model has built, the origin is handled as usual: after a migration, the old name holds a compatibility view, which expires as described above. + ## Relation age `retention_days` is measured from the last time a relation was written, as reported by the warehouse. For a relation the project no longer builds, that is roughly when it stopped being built. diff --git a/website/src/content/docs/docs/cli/rename.mdx b/website/src/content/docs/docs/cli/rename.mdx new file mode 100644 index 0000000000..9230251a2a --- /dev/null +++ b/website/src/content/docs/docs/cli/rename.mdx @@ -0,0 +1,112 @@ +--- +title: "rename and mv" +description: "Rename or move a model, or rename a column, and update every reference in one verified step." +--- + +`sqb rename` and `sqb mv` change a name everywhere the compiler knows it is used, instead of leaving you to find every `__ref`, fixture, and header by hand. Every edit comes from compiled facts, not text search: a column rename follows the columns each query actually binds, so a same-named column of another table is left alone. + +Nothing is written until the edited project compiles. The command applies its edits to a scratch copy of the project, compiles it the way `sqb compile` does, and only then writes the files. If anything cannot be rewritten safely, the command lists those locations and writes nothing. + +## Usage + +```bash +sqb rename model: [flags] +sqb mv model: [flags] +sqb rename column:. [--cascade] [flags] +``` + +```bash +sqb rename model:stg_orders stg_order_lines +sqb mv model:fact_orders models/reporting/ # keeps the name +sqb mv model:fact_orders models/reporting/order_facts.sql # moves and renames +sqb rename column:stg_orders.amount revenue +sqb rename column:stg_orders.amount revenue --cascade +``` + +A model rename keeps the file in its folder and renames it to `.sql`. `sqb mv` takes a new file path, which also sets the model name, or a folder ending in `/`, which keeps it. + +## What changes + +Renaming or moving a model rewrites: + +- every `__ref("")` in models, tests, scenarios, audits, hooks, and SQL functions; +- `__ref__` and `__expected__` fixture CTEs in tests and scenarios; +- `cursor_inputs` keys, `relationships (to ...)` audits, and `__ref("")` inside quoted SQL in `MODEL` headers and reusable `SCHEMA` declarations; +- `relationships` audits (`to:` and `field:`) and `__ref("")` in source and seed YAML. + +Moving a model to another folder also moves the macros, enums, constants, reusable schemas, generic audits, and named hooks it uses when their [placement](/docs/concepts/declaration-scopes/placement) must change, following the same rules `sqb compile` enforces: + +- a declaration only the moved model uses moves with it, into the destination folder's matching role; +- a declaration other resources still use moves to the lowest folder they share, for example `models/staging/_sqlbuild/macros/`; +- when that folder would be an authored root such as `models/`, or the users span resource trees such as models and tests, it moves to the top-level role (`macros/`, `enums/`, `constants/`, `schemas/`, ...). + +Declarations are moved, never copied. Folders the move leaves empty are removed. + +Renaming a column rewrites the model that produces it and every query that reads it: + +- the producing projection, as `amount AS revenue`, or the existing alias; +- every downstream reference, including filters, joins, grouping, and ordering; +- column declarations and column-valued config such as `unique_key`, `cursor`, and `partition_by`, and `cursor_inputs` and `relationships (field ...)` entries that point at the column; +- fixture columns of `__ref__` and `__expected__` CTEs. + +By default a column rename is one step: downstream models keep their output names, so `SELECT o.amount` becomes `SELECT o.revenue AS amount`. With `--cascade`, a downstream model that passes the column through unchanged, as a bare column, a same-named alias, or `SELECT *`, renames its output too, and the rename continues through its own consumers. + +## History + +A renamed model keeps its warehouse history through a [model migration](/docs/concepts/models/migrations). Whenever the model's relation moves and its materialization keeps warehouse data (table, view, incremental, or snapshot), the command adds `migrate_from ` to the model header, so the move does not depend on automatic discovery finding exactly one match. The next `sqb build` migrates the relation and leaves a compatibility view at the old name. A move that keeps the relation, such as a folder change in the same schema, adds nothing. + +On a target that never built the old name, the origin is missing. By default the plan warns and the model builds fresh; set [`missing_migration_origin`](/docs/concepts/project-configuration#missing-migration-origins) on a target to require confirmation or stop the build instead. + +A renamed column of an incremental or snapshot model gets ` (migrate_from )` in its `columns` block, so the next build renames it in place as a [column migration](/docs/concepts/models/column-migrations). Table and view models are rebuilt with the new column anyway; they get the declaration only when the model itself declares `migrate_from`. + +The command refuses a rename while the model, or the column, still declares `migrate_from` from an earlier rename. Build it on every target and remove the declaration first. + +## Refusals + +The command refuses, and writes nothing, when: + +- the new name already belongs to another model or column, or the destination file exists; +- a declaration a moved model uses must change folder, but its file also holds declarations that must stay, or it is not an authored project file, or a file with its name already exists at the new folder; +- a move would leave no model in the old schema, so the old relation could not be found; +- a reference is produced by a macro, or a column is read inside SQL that a macro generates; +- a column reaches a model through `SELECT *` in a subquery, a `USING` or `NATURAL` join, or an unqualified name that could belong to several tables; +- a downstream model's `SELECT *` would silently rename its output, without `--cascade`; +- Python code or Python node SQL strings name the model or column. + +Each location is listed with its file, line, and column, with the fix. Edit those locations, or pass the model or column to the macro as an argument, and run the command again. The command never applies part of a rename. + +References produced by a macro are only possible when a project turns off explicit references (`[references] enforce_explicit = false`); such projects get the refusal list for them. + +## Flags + +| Flag | Description | +|------|-------------| +| `--cascade` | For column renames, rename the output of every downstream model that passes the column through. | +| `--dry-run` | Print every edit and verify the edited project compiles, without writing files. | +| `--json` | Print the result as JSON on stdout. Progress goes to stderr. | + +## Output + +```text +Rename model stg_orders -> stg_order_lines +├── models/marts/fact_orders.sql +│ └── 24:13 reference __ref("stg_orders") -> __ref("stg_order_lines") +├── models/staging/stg_order_lines.sql (moved from models/staging/stg_orders.sql) +└── tests/unit/test_fact_orders.sql + └── 4:1 fixture __ref__stg_orders -> __ref__stg_order_lines +Compiled: ok, 3 files changed +``` + +With `--json`, the result has `operation`, `target`, `new_name`, `destination`, `status` (`applied`, `dry_run`, `refused`, or `compile_failed`), `files` with every edit, `migrations`, `renamed_columns`, `manual`, `blocking`, and `compile.errors`. + +## Exit codes + +| Code | Meaning | +|------|---------| +| `0` | Applied, or a dry run that compiles. | +| `1` | Refused, the edited project did not compile, or the request was invalid. No files were changed. | + +## Limitations + +- Descriptions and comments are not rewritten. +- Queries outside the project that read the old relation keep working through the old-name compatibility view until it expires. diff --git a/website/src/content/docs/docs/concepts/models/column-migrations.mdx b/website/src/content/docs/docs/concepts/models/column-migrations.mdx index 7cc03397f1..4fb598e3a2 100644 --- a/website/src/content/docs/docs/concepts/models/column-migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/column-migrations.mdx @@ -9,6 +9,8 @@ Column migrations apply to incremental models, including microbatch models, and ## Declaring a rename +[`sqb rename column:.`](/docs/cli/rename) renames the column in the model and every query that reads it, and adds the `migrate_from` declaration below for you. + Add `migrate_from` to the renamed column in the model header, naming the old column: ```sql @@ -78,7 +80,7 @@ Automatically detected renames are marked `(automatic)`. The decision is based o | `already renamed, record` | The new column exists and the old one doesn't, but no rename is recorded, which is what an interrupted rename leaves behind. Record it without renaming again. | | `done` | The rename is recorded and the new column exists. Nothing to do. | | `conflict` | Both columns exist. The build stops (`M110`); drop one of them, or remove `migrate_from` and let `on_schema_change` handle the columns. | -| `source missing` | Neither column exists and no rename is recorded. The build stops (`M109`); remove `migrate_from` if the rename happened elsewhere or is no longer needed. | +| `source missing` | Neither column exists and no rename is recorded. The plan warns (`M109`) and the columns follow `on_schema_change` as if `migrate_from` were absent. A target can require confirmation or stop the build instead with [`missing_migration_origin`](/docs/concepts/project-configuration#missing-migration-origins). | | `source still produced` | The query still selects the old column. The build stops (`M111`). | | `unsupported` | The warehouse can't rename this table's columns in place (see [Adapter support](#adapter-support)). The build stops (`M112`). | diff --git a/website/src/content/docs/docs/concepts/models/migrations.mdx b/website/src/content/docs/docs/concepts/models/migrations.mdx index 31388aec32..fde311af7c 100644 --- a/website/src/content/docs/docs/concepts/models/migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/migrations.mdx @@ -9,7 +9,9 @@ Table and view models can declare a migration too. They hold no history, so they ## Declaring a migration -Add `migrate_from` to the renamed model's header, naming the old model: +[`sqb rename`](/docs/cli/rename) and `sqb mv` rename or move a model, update every reference to it, and add `migrate_from` whenever the model's relation moves and its materialization keeps warehouse data (table, view, incremental, or snapshot). + +To declare a migration by hand, add `migrate_from` to the renamed model's header, naming the old model: ```sql MODEL ( @@ -38,7 +40,7 @@ After the move is recorded, `migrate_from` has no further effect, and SQLBuild t | `superseded replace` | The new relation was itself the origin of the latest recorded move, for example after renaming a model back. Replace it with the newer data. | | `forced replace` | The new relation has its own build history and `migrate_force true` is set. Replace it. | | `conflict` | The new relation has its own build history. The build stops (`M103`) until you set `migrate_force true` or remove `migrate_from`. | -| `origin missing` | The old relation doesn't exist and no move was recorded. The build stops (`M102`). | +| `origin missing` | The old relation doesn't exist and no move was recorded, for example on a target that never built the old name. The plan warns (`M102`) and the model builds fresh. A target can require confirmation or stop the build instead with [`missing_migration_origin`](/docs/concepts/project-configuration#missing-migration-origins). If SQLBuild has build history for the old relation in this target, it should exist, so the build stops (`M102`) whatever the policy: the connection may lack permission to see it, or it was dropped outside SQLBuild. Restore access, or remove `migrate_from` to build fresh deliberately. | The origin and destination must be compatible under the model's normal `on_schema_change` rules; otherwise the build stops (`M104`). diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 4af4b22801..1a97666da6 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -129,6 +129,7 @@ A target is a named build context - the database and schema you build into, plus | `vars` | Target-specific project variables | | `defer_sources_to` | Target name to read managed source data from (see [Loaders](/docs/concepts/python-nodes/loaders#source-deferral)) | | `clone` | Clone policy (see below) | +| `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny` (see below) | ```toml [connections.warehouse] @@ -194,6 +195,25 @@ password = "${ENV:SNOWFLAKE_PASSWORD}" Commands then use `dev` automatically. An explicit command such as `sqb build --target prod` still takes precedence for that invocation. +### Missing migration origins + +A `migrate_from` declaration, on a model or a column, names a relation or column that exists in the targets where the rename is pending. A target that never built the old name, such as a fresh development schema, has nothing to migrate. `missing_migration_origin` decides what happens there: + +| Value | Behavior | +|-------|----------| +| `allow` | Default. The plan warns (`M102` for models, `M109` for columns), nothing is migrated, and the model builds as if `migrate_from` were absent. | +| `require_confirmation` | The build asks for confirmation on a terminal, and stops otherwise; pass `--allow-missing-migration-origin` to confirm in non-interactive runs. | +| `deny` | The build stops with `M102` or `M109`. Use it for targets where a missing origin means the wrong target or a lost relation. | + +```toml +[targets.prod] +missing_migration_origin = "deny" +``` + +`sqlbuild_local.toml` can override the value for a target. + +The policy covers only origins with no build history in the target. A model origin that SQLBuild built in this target but can no longer see, because of permissions or because it was dropped outside SQLBuild, always stops the build with `M102`. Column origins have no such history, so `M109` always follows the policy. `sqb dbt run` and `sqb dbt build` apply the policy too, but have no confirmation flag: they ask on a terminal and stop otherwise. In non-interactive dbt runs, build the affected models with `sqb build --allow-missing-migration-origin` and leave them out of the dbt selection, or set the target's `missing_migration_origin` to `allow`. + ### Clone policies Targets can declare whether they allow cloning to or from: diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 487d278a8b..98bdf37bb5 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -188,6 +188,7 @@ "docs/cli/format", "docs/cli/contract", "docs/cli/scope", + "docs/cli/rename", "docs/cli/rules", "docs/cli/plan", "docs/cli/build",